mirror of
https://github.com/Kaelio/ktx.git
synced 2026-06-13 08:15:14 +02:00
Initial open-source release
This commit is contained in:
commit
1a42152e6f
1199 changed files with 257054 additions and 0 deletions
108
packages/cli/src/agent-runtime.test.ts
Normal file
108
packages/cli/src/agent-runtime.test.ts
Normal file
|
|
@ -0,0 +1,108 @@
|
|||
import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import {
|
||||
KLO_AGENT_MAX_ROWS_CAP,
|
||||
createKloAgentRuntime,
|
||||
parseAgentMaxRows,
|
||||
readAgentJsonFile,
|
||||
writeAgentJson,
|
||||
writeAgentJsonError,
|
||||
} from './agent-runtime.js';
|
||||
|
||||
function makeIo() {
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
return {
|
||||
io: {
|
||||
stdout: { write: (chunk: string) => (stdout += chunk) },
|
||||
stderr: { write: (chunk: string) => (stderr += chunk) },
|
||||
},
|
||||
stdout: () => stdout,
|
||||
stderr: () => stderr,
|
||||
};
|
||||
}
|
||||
|
||||
describe('agent runtime helpers', () => {
|
||||
let tempDir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'klo-agent-runtime-'));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(tempDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('writes JSON success and error envelopes without color or spinners', () => {
|
||||
const successIo = makeIo();
|
||||
const errorIo = makeIo();
|
||||
|
||||
writeAgentJson(successIo.io, { ok: true });
|
||||
writeAgentJsonError(errorIo.io, 'missing source', { code: 'NOT_FOUND' });
|
||||
|
||||
expect(JSON.parse(successIo.stdout())).toEqual({ ok: true });
|
||||
expect(successIo.stderr()).toBe('');
|
||||
expect(JSON.parse(errorIo.stderr())).toEqual({
|
||||
ok: false,
|
||||
error: { message: 'missing source', code: 'NOT_FOUND' },
|
||||
});
|
||||
expect(errorIo.stdout()).toBe('');
|
||||
});
|
||||
|
||||
it('reads JSON query files as objects', async () => {
|
||||
const path = join(tempDir, 'query.json');
|
||||
await writeFile(path, '{"measures":["revenue"],"limit":50}', 'utf-8');
|
||||
|
||||
await expect(readAgentJsonFile(path)).resolves.toEqual({ measures: ['revenue'], limit: 50 });
|
||||
});
|
||||
|
||||
it('rejects non-object JSON query files', async () => {
|
||||
const path = join(tempDir, 'query.json');
|
||||
await writeFile(path, '["revenue"]', 'utf-8');
|
||||
|
||||
await expect(readAgentJsonFile(path)).rejects.toThrow('must contain a JSON object');
|
||||
});
|
||||
|
||||
it('requires positive row limits and enforces the agent cap', () => {
|
||||
expect(parseAgentMaxRows(100)).toBe(100);
|
||||
expect(() => parseAgentMaxRows(undefined)).toThrow('maxRows is required');
|
||||
expect(() => parseAgentMaxRows(0)).toThrow('positive integer');
|
||||
expect(() => parseAgentMaxRows(KLO_AGENT_MAX_ROWS_CAP + 1)).toThrow(String(KLO_AGENT_MAX_ROWS_CAP));
|
||||
});
|
||||
|
||||
it('constructs local context ports with semantic compute and query executor', async () => {
|
||||
const project = {
|
||||
projectDir: tempDir,
|
||||
configPath: join(tempDir, 'klo.yaml'),
|
||||
config: { project: 'revenue', connections: {} },
|
||||
coreConfig: {},
|
||||
git: {},
|
||||
fileStore: {},
|
||||
} as never;
|
||||
const ports = { knowledge: {}, semanticLayer: {} } as never;
|
||||
const semanticLayerCompute = { query: vi.fn(), validateSources: vi.fn(), generateSources: vi.fn() };
|
||||
const queryExecutor = { execute: vi.fn() };
|
||||
const loadProject = vi.fn(async () => project);
|
||||
const createContextTools = vi.fn(() => ports);
|
||||
|
||||
await expect(
|
||||
createKloAgentRuntime(
|
||||
{ projectDir: tempDir, enableSemanticCompute: true, enableQueryExecution: true },
|
||||
{
|
||||
loadProject,
|
||||
createContextTools,
|
||||
createSemanticLayerCompute: () => semanticLayerCompute,
|
||||
createQueryExecutor: () => queryExecutor,
|
||||
},
|
||||
),
|
||||
).resolves.toMatchObject({ project, ports, queryExecutor });
|
||||
|
||||
expect(loadProject).toHaveBeenCalledWith({ projectDir: tempDir });
|
||||
expect(createContextTools).toHaveBeenCalledWith(project, {
|
||||
semanticLayerCompute,
|
||||
queryExecutor,
|
||||
});
|
||||
});
|
||||
});
|
||||
81
packages/cli/src/agent-runtime.ts
Normal file
81
packages/cli/src/agent-runtime.ts
Normal file
|
|
@ -0,0 +1,81 @@
|
|||
import { readFile } from 'node:fs/promises';
|
||||
import { createDefaultLocalQueryExecutor, type KloSqlQueryExecutorPort } from '@klo/context/connections';
|
||||
import { createPythonSemanticLayerComputePort, type KloSemanticLayerComputePort } from '@klo/context/daemon';
|
||||
import { createLocalProjectMcpContextPorts, type KloMcpContextPorts } from '@klo/context/mcp';
|
||||
import { type KloLocalProject, loadKloProject } from '@klo/context/project';
|
||||
import type { KloCliIo } from './cli-runtime.js';
|
||||
|
||||
export const KLO_AGENT_MAX_ROWS_CAP = 1000;
|
||||
|
||||
export interface KloAgentRuntimeOptions {
|
||||
projectDir: string;
|
||||
enableSemanticCompute: boolean;
|
||||
enableQueryExecution: boolean;
|
||||
}
|
||||
|
||||
export interface KloAgentRuntime {
|
||||
project: KloLocalProject;
|
||||
ports: KloMcpContextPorts;
|
||||
semanticLayerCompute?: KloSemanticLayerComputePort;
|
||||
queryExecutor?: KloSqlQueryExecutorPort;
|
||||
}
|
||||
|
||||
export interface KloAgentRuntimeDeps {
|
||||
loadProject?: typeof loadKloProject;
|
||||
createContextTools?: typeof createLocalProjectMcpContextPorts;
|
||||
createSemanticLayerCompute?: () => KloSemanticLayerComputePort;
|
||||
createQueryExecutor?: () => KloSqlQueryExecutorPort;
|
||||
}
|
||||
|
||||
export function writeAgentJson(io: KloCliIo, value: unknown): void {
|
||||
io.stdout.write(`${JSON.stringify(value, null, 2)}\n`);
|
||||
}
|
||||
|
||||
export function writeAgentJsonError(
|
||||
io: KloCliIo,
|
||||
message: string,
|
||||
detail: Record<string, unknown> = {},
|
||||
): void {
|
||||
io.stderr.write(`${JSON.stringify({ ok: false, error: { message, ...detail } }, null, 2)}\n`);
|
||||
}
|
||||
|
||||
export async function readAgentJsonFile(path: string): Promise<Record<string, unknown>> {
|
||||
const parsed = JSON.parse(await readFile(path, 'utf-8')) as unknown;
|
||||
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
|
||||
throw new Error(`${path} must contain a JSON object.`);
|
||||
}
|
||||
return parsed as Record<string, unknown>;
|
||||
}
|
||||
|
||||
export function parseAgentMaxRows(value: number | undefined): number {
|
||||
if (!Number.isInteger(value) || value === undefined || value <= 0) {
|
||||
throw new Error('maxRows is required and must be a positive integer.');
|
||||
}
|
||||
if (value > KLO_AGENT_MAX_ROWS_CAP) {
|
||||
throw new Error(`maxRows must be less than or equal to ${KLO_AGENT_MAX_ROWS_CAP}.`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
export async function createKloAgentRuntime(
|
||||
options: KloAgentRuntimeOptions,
|
||||
deps: KloAgentRuntimeDeps = {},
|
||||
): Promise<KloAgentRuntime> {
|
||||
const project = await (deps.loadProject ?? loadKloProject)({ projectDir: options.projectDir });
|
||||
const semanticLayerCompute = options.enableSemanticCompute
|
||||
? (deps.createSemanticLayerCompute ?? createPythonSemanticLayerComputePort)()
|
||||
: undefined;
|
||||
const queryExecutor = options.enableQueryExecution
|
||||
? (deps.createQueryExecutor ?? createDefaultLocalQueryExecutor)()
|
||||
: undefined;
|
||||
const ports = (deps.createContextTools ?? createLocalProjectMcpContextPorts)(project, {
|
||||
...(semanticLayerCompute ? { semanticLayerCompute } : {}),
|
||||
...(queryExecutor ? { queryExecutor } : {}),
|
||||
});
|
||||
return {
|
||||
project,
|
||||
ports,
|
||||
...(semanticLayerCompute ? { semanticLayerCompute } : {}),
|
||||
...(queryExecutor ? { queryExecutor } : {}),
|
||||
};
|
||||
}
|
||||
51
packages/cli/src/agent-search-readiness.test.ts
Normal file
51
packages/cli/src/agent-search-readiness.test.ts
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
isMissingProjectConfigError,
|
||||
missingConnectionSlSearchReadiness,
|
||||
missingProjectSlSearchReadiness,
|
||||
noConnectionsSlSearchReadiness,
|
||||
noIndexedSourcesSlSearchReadiness,
|
||||
} from './agent-search-readiness.js';
|
||||
|
||||
describe('agent semantic-layer search readiness guidance', () => {
|
||||
it('formats missing project guidance with exact recovery commands', () => {
|
||||
expect(missingProjectSlSearchReadiness('/tmp/klo-search', 'gross revenue')).toEqual({
|
||||
code: 'agent_sl_search_missing_project',
|
||||
message: 'Semantic-layer search needs an initialized KLO project at /tmp/klo-search.',
|
||||
nextSteps: [
|
||||
'klo demo',
|
||||
'klo setup --project-dir /tmp/klo-search',
|
||||
'klo ingest <connection>',
|
||||
'klo agent sl list --json --query "gross revenue" --project-dir /tmp/klo-search',
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it('formats no-connection and no-index guidance without hiding the project path', () => {
|
||||
expect(noConnectionsSlSearchReadiness('/tmp/klo-search', 'revenue')).toMatchObject({
|
||||
code: 'agent_sl_search_no_connections',
|
||||
message: 'Semantic-layer search found no configured connections in /tmp/klo-search.',
|
||||
});
|
||||
expect(noIndexedSourcesSlSearchReadiness('/tmp/klo-search', 'orders')).toMatchObject({
|
||||
code: 'agent_sl_search_no_indexed_sources',
|
||||
message: 'Semantic-layer search found no indexed semantic-layer sources in /tmp/klo-search.',
|
||||
});
|
||||
});
|
||||
|
||||
it('formats unknown connection guidance', () => {
|
||||
expect(missingConnectionSlSearchReadiness('/tmp/klo-search', 'warehouse', 'revenue')).toMatchObject({
|
||||
code: 'agent_sl_search_unknown_connection',
|
||||
message: 'Semantic-layer search connection "warehouse" is not configured in /tmp/klo-search.',
|
||||
});
|
||||
});
|
||||
|
||||
it('detects missing klo.yaml read errors', () => {
|
||||
const error = Object.assign(new Error('ENOENT: no such file or directory'), {
|
||||
code: 'ENOENT',
|
||||
path: '/tmp/klo-search/klo.yaml',
|
||||
});
|
||||
|
||||
expect(isMissingProjectConfigError(error)).toBe(true);
|
||||
expect(isMissingProjectConfigError(new Error('other'))).toBe(false);
|
||||
});
|
||||
});
|
||||
94
packages/cli/src/agent-search-readiness.ts
Normal file
94
packages/cli/src/agent-search-readiness.ts
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
export type KloAgentSlSearchReadinessCode =
|
||||
| 'agent_sl_search_missing_project'
|
||||
| 'agent_sl_search_no_connections'
|
||||
| 'agent_sl_search_unknown_connection'
|
||||
| 'agent_sl_search_no_indexed_sources';
|
||||
|
||||
export interface KloAgentSlSearchReadinessDetail {
|
||||
code: KloAgentSlSearchReadinessCode;
|
||||
message: string;
|
||||
nextSteps: string[];
|
||||
}
|
||||
|
||||
function queryForCommand(query: string | undefined): string {
|
||||
const trimmed = query?.trim();
|
||||
return trimmed && trimmed.length > 0 ? trimmed : 'revenue';
|
||||
}
|
||||
|
||||
function projectSearchCommand(projectDir: string, query: string | undefined): string {
|
||||
return `klo agent sl list --json --query ${JSON.stringify(queryForCommand(query))} --project-dir ${projectDir}`;
|
||||
}
|
||||
|
||||
function baseNextSteps(projectDir: string, query: string | undefined): string[] {
|
||||
return [
|
||||
'klo demo',
|
||||
`klo setup --project-dir ${projectDir}`,
|
||||
'klo ingest <connection>',
|
||||
projectSearchCommand(projectDir, query),
|
||||
];
|
||||
}
|
||||
|
||||
export function missingProjectSlSearchReadiness(
|
||||
projectDir: string,
|
||||
query: string | undefined,
|
||||
): KloAgentSlSearchReadinessDetail {
|
||||
return {
|
||||
code: 'agent_sl_search_missing_project',
|
||||
message: `Semantic-layer search needs an initialized KLO project at ${projectDir}.`,
|
||||
nextSteps: baseNextSteps(projectDir, query),
|
||||
};
|
||||
}
|
||||
|
||||
export function noConnectionsSlSearchReadiness(
|
||||
projectDir: string,
|
||||
query: string | undefined,
|
||||
): KloAgentSlSearchReadinessDetail {
|
||||
return {
|
||||
code: 'agent_sl_search_no_connections',
|
||||
message: `Semantic-layer search found no configured connections in ${projectDir}.`,
|
||||
nextSteps: baseNextSteps(projectDir, query),
|
||||
};
|
||||
}
|
||||
|
||||
export function missingConnectionSlSearchReadiness(
|
||||
projectDir: string,
|
||||
connectionId: string,
|
||||
query: string | undefined,
|
||||
): KloAgentSlSearchReadinessDetail {
|
||||
return {
|
||||
code: 'agent_sl_search_unknown_connection',
|
||||
message: `Semantic-layer search connection "${connectionId}" is not configured in ${projectDir}.`,
|
||||
nextSteps: baseNextSteps(projectDir, query),
|
||||
};
|
||||
}
|
||||
|
||||
export function noIndexedSourcesSlSearchReadiness(
|
||||
projectDir: string,
|
||||
query: string | undefined,
|
||||
): KloAgentSlSearchReadinessDetail {
|
||||
return {
|
||||
code: 'agent_sl_search_no_indexed_sources',
|
||||
message: `Semantic-layer search found no indexed semantic-layer sources in ${projectDir}.`,
|
||||
nextSteps: baseNextSteps(projectDir, query),
|
||||
};
|
||||
}
|
||||
|
||||
function errorCode(error: unknown): string | undefined {
|
||||
if (typeof error !== 'object' || error === null || !('code' in error)) {
|
||||
return undefined;
|
||||
}
|
||||
const code = (error as { code?: unknown }).code;
|
||||
return typeof code === 'string' ? code : undefined;
|
||||
}
|
||||
|
||||
function errorPath(error: unknown): string | undefined {
|
||||
if (typeof error !== 'object' || error === null || !('path' in error)) {
|
||||
return undefined;
|
||||
}
|
||||
const path = (error as { path?: unknown }).path;
|
||||
return typeof path === 'string' ? path : undefined;
|
||||
}
|
||||
|
||||
export function isMissingProjectConfigError(error: unknown): boolean {
|
||||
return errorCode(error) === 'ENOENT' && (errorPath(error)?.endsWith('klo.yaml') ?? false);
|
||||
}
|
||||
393
packages/cli/src/agent.test.ts
Normal file
393
packages/cli/src/agent.test.ts
Normal file
|
|
@ -0,0 +1,393 @@
|
|||
import { mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { buildDefaultKloProjectConfig } from '@klo/context/project';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { runKloAgent } from './agent.js';
|
||||
import type { KloAgentRuntime } from './agent-runtime.js';
|
||||
|
||||
function makeIo() {
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
return {
|
||||
io: {
|
||||
stdout: { write: (chunk: string) => (stdout += chunk) },
|
||||
stderr: { write: (chunk: string) => (stderr += chunk) },
|
||||
},
|
||||
stdout: () => stdout,
|
||||
stderr: () => stderr,
|
||||
};
|
||||
}
|
||||
|
||||
function runtime(overrides: Record<string, unknown> = {}): KloAgentRuntime {
|
||||
const config = buildDefaultKloProjectConfig('revenue');
|
||||
return {
|
||||
project: {
|
||||
projectDir: '/tmp/revenue',
|
||||
configPath: '/tmp/revenue/klo.yaml',
|
||||
config: {
|
||||
...config,
|
||||
connections: {
|
||||
warehouse: { driver: 'sqlite', path: 'warehouse.sqlite', readonly: true as const },
|
||||
},
|
||||
},
|
||||
coreConfig: {} as KloAgentRuntime['project']['coreConfig'],
|
||||
git: {} as KloAgentRuntime['project']['git'],
|
||||
fileStore: {} as KloAgentRuntime['project']['fileStore'],
|
||||
},
|
||||
ports: {
|
||||
connections: { list: vi.fn(async () => [{ id: 'warehouse', name: 'warehouse', connectionType: 'sqlite' }]) },
|
||||
semanticLayer: {
|
||||
listSources: vi.fn(async () => ({
|
||||
sources: [
|
||||
{
|
||||
connectionId: 'warehouse',
|
||||
connectionName: 'warehouse',
|
||||
name: 'orders',
|
||||
columnCount: 2,
|
||||
measureCount: 1,
|
||||
joinCount: 0,
|
||||
},
|
||||
],
|
||||
totalSources: 1,
|
||||
})),
|
||||
readSource: vi.fn(async () => ({ sourceName: 'orders', yaml: 'name: orders\n' })),
|
||||
writeSource: vi.fn(async () => ({ success: true, sourceName: 'orders' })),
|
||||
validate: vi.fn(async () => ({ success: true, errors: [], warnings: [] })),
|
||||
query: vi.fn(async () => ({ sql: 'select 1', headers: ['x'], rows: [[1]], totalRows: 1, plan: {} })),
|
||||
},
|
||||
knowledge: {
|
||||
search: vi.fn(async () => ({
|
||||
results: [
|
||||
{
|
||||
key: 'page-1',
|
||||
path: 'knowledge/global/page-1.md',
|
||||
scope: 'GLOBAL' as const,
|
||||
summary: 'Revenue logic',
|
||||
score: 0.9,
|
||||
matchReasons: ['lexical' as const],
|
||||
},
|
||||
],
|
||||
totalFound: 1,
|
||||
})),
|
||||
read: vi.fn(async () => ({
|
||||
key: 'page-1',
|
||||
scope: 'GLOBAL' as const,
|
||||
summary: 'Revenue logic',
|
||||
content: 'Use net revenue.',
|
||||
})),
|
||||
write: vi.fn(async () => ({ success: true, key: 'page-1', action: 'created' as const })),
|
||||
},
|
||||
},
|
||||
queryExecutor: {
|
||||
execute: vi.fn(async () => ({ headers: ['x'], rows: [[1]], totalRows: 1, command: 'SELECT', rowCount: 1 })),
|
||||
},
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function runtimeWithoutConnections(): KloAgentRuntime {
|
||||
const base = runtime();
|
||||
return {
|
||||
...base,
|
||||
project: {
|
||||
...base.project,
|
||||
config: {
|
||||
...base.project.config,
|
||||
connections: {},
|
||||
},
|
||||
},
|
||||
ports: {
|
||||
...base.ports,
|
||||
semanticLayer: {
|
||||
...base.ports.semanticLayer!,
|
||||
listSources: vi.fn(async () => ({ sources: [], totalSources: 0 })),
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe('runKloAgent', () => {
|
||||
let tempDir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'klo-agent-'));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(tempDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('prints tool discovery with every stable command', async () => {
|
||||
const io = makeIo();
|
||||
|
||||
await expect(runKloAgent({ command: 'tools', projectDir: tempDir, json: true }, io.io)).resolves.toBe(0);
|
||||
|
||||
const body = JSON.parse(io.stdout());
|
||||
expect(body.projectDir).toBe(tempDir);
|
||||
expect(body.tools.map((tool: { name: string }) => tool.name)).toEqual([
|
||||
'context',
|
||||
'sl.list',
|
||||
'sl.read',
|
||||
'sl.query',
|
||||
'wiki.search',
|
||||
'wiki.read',
|
||||
'sql.execute',
|
||||
]);
|
||||
expect(io.stderr()).toBe('');
|
||||
});
|
||||
|
||||
it('prints project context from setup status, connections, and SL summaries', async () => {
|
||||
const io = makeIo();
|
||||
const createRuntime = vi.fn(async () => runtime());
|
||||
const readSetupStatus = vi.fn(async () => ({ project: { path: tempDir, ready: true }, agents: [] }));
|
||||
|
||||
await expect(
|
||||
runKloAgent({ command: 'context', projectDir: tempDir, json: true }, io.io, { createRuntime, readSetupStatus }),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(JSON.parse(io.stdout())).toMatchObject({
|
||||
projectDir: tempDir,
|
||||
status: { project: { ready: true } },
|
||||
connections: [{ id: 'warehouse' }],
|
||||
semanticLayer: { totalSources: 1 },
|
||||
});
|
||||
});
|
||||
|
||||
it('dispatches SL list, SL read, wiki search, and wiki read through local ports', async () => {
|
||||
for (const args of [
|
||||
{ command: 'sl-list' as const, projectDir: tempDir, json: true as const, connectionId: 'warehouse' },
|
||||
{
|
||||
command: 'sl-read' as const,
|
||||
projectDir: tempDir,
|
||||
json: true as const,
|
||||
connectionId: 'warehouse',
|
||||
sourceName: 'orders',
|
||||
},
|
||||
{ command: 'wiki-search' as const, projectDir: tempDir, json: true as const, query: 'revenue', limit: 10 },
|
||||
{ command: 'wiki-read' as const, projectDir: tempDir, json: true as const, pageId: 'page-1' },
|
||||
]) {
|
||||
const io = makeIo();
|
||||
await expect(runKloAgent(args, io.io, { createRuntime: async () => runtime() })).resolves.toBe(0);
|
||||
expect(JSON.parse(io.stdout())).toBeTruthy();
|
||||
expect(io.stderr()).toBe('');
|
||||
}
|
||||
});
|
||||
|
||||
it('prints wiki hybrid search metadata from the hidden agent wiki search command', async () => {
|
||||
const fakeRuntime = runtime();
|
||||
const knowledge = fakeRuntime.ports.knowledge;
|
||||
if (!knowledge) {
|
||||
throw new Error('Expected runtime knowledge port');
|
||||
}
|
||||
fakeRuntime.ports.knowledge = {
|
||||
...knowledge,
|
||||
search: vi.fn(async () => ({
|
||||
results: [
|
||||
{
|
||||
key: 'metrics/revenue',
|
||||
path: 'knowledge/global/metrics/revenue.md',
|
||||
scope: 'GLOBAL' as const,
|
||||
summary: 'Revenue metric definition',
|
||||
score: 0.02459016393442623,
|
||||
matchReasons: ['lexical' as const, 'token' as const],
|
||||
},
|
||||
],
|
||||
totalFound: 1,
|
||||
})),
|
||||
};
|
||||
const io = makeIo();
|
||||
|
||||
await expect(
|
||||
runKloAgent({ command: 'wiki-search', projectDir: tempDir, json: true, query: 'paid order', limit: 5 }, io.io, {
|
||||
createRuntime: async () => fakeRuntime,
|
||||
}),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(JSON.parse(io.stdout())).toEqual({
|
||||
results: [
|
||||
expect.objectContaining({
|
||||
key: 'metrics/revenue',
|
||||
path: 'knowledge/global/metrics/revenue.md',
|
||||
matchReasons: ['lexical', 'token'],
|
||||
}),
|
||||
],
|
||||
totalFound: 1,
|
||||
});
|
||||
});
|
||||
|
||||
it('executes SL queries from a JSON query file', async () => {
|
||||
const queryFile = join(tempDir, 'sl-query.json');
|
||||
const io = makeIo();
|
||||
await writeFile(queryFile, '{"measures":["total_revenue"],"dimensions":[]}', 'utf-8');
|
||||
|
||||
await expect(
|
||||
runKloAgent(
|
||||
{
|
||||
command: 'sl-query',
|
||||
projectDir: tempDir,
|
||||
json: true,
|
||||
connectionId: 'warehouse',
|
||||
queryFile,
|
||||
execute: true,
|
||||
maxRows: 100,
|
||||
},
|
||||
io.io,
|
||||
{ createRuntime: async () => runtime() },
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(JSON.parse(io.stdout())).toMatchObject({ sql: 'select 1', rows: [[1]] });
|
||||
});
|
||||
|
||||
it('executes read-only SQL from a SQL file with an explicit row limit', async () => {
|
||||
const sqlFile = join(tempDir, 'query.sql');
|
||||
const fakeRuntime = runtime();
|
||||
const io = makeIo();
|
||||
await writeFile(sqlFile, 'select 1', 'utf-8');
|
||||
|
||||
await expect(
|
||||
runKloAgent(
|
||||
{
|
||||
command: 'sql-execute',
|
||||
projectDir: tempDir,
|
||||
json: true,
|
||||
connectionId: 'warehouse',
|
||||
sqlFile,
|
||||
maxRows: 100,
|
||||
},
|
||||
io.io,
|
||||
{ createRuntime: async () => fakeRuntime as never },
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(fakeRuntime.queryExecutor?.execute).toHaveBeenCalledWith({
|
||||
connectionId: 'warehouse',
|
||||
projectDir: '/tmp/revenue',
|
||||
connection: { driver: 'sqlite', path: 'warehouse.sqlite', readonly: true },
|
||||
sql: 'select 1',
|
||||
maxRows: 100,
|
||||
});
|
||||
});
|
||||
|
||||
it('prints guided JSON when semantic-layer search runs outside a project', async () => {
|
||||
const io = makeIo();
|
||||
const missingProjectError = Object.assign(new Error('ENOENT: no such file or directory'), {
|
||||
code: 'ENOENT',
|
||||
path: join(tempDir, 'klo.yaml'),
|
||||
});
|
||||
|
||||
await expect(
|
||||
runKloAgent(
|
||||
{ command: 'sl-list', projectDir: tempDir, json: true, query: 'gross revenue' },
|
||||
io.io,
|
||||
{ createRuntime: vi.fn(async () => Promise.reject(missingProjectError)) },
|
||||
),
|
||||
).resolves.toBe(1);
|
||||
|
||||
expect(JSON.parse(io.stderr())).toEqual({
|
||||
ok: false,
|
||||
error: {
|
||||
code: 'agent_sl_search_missing_project',
|
||||
message: `Semantic-layer search needs an initialized KLO project at ${tempDir}.`,
|
||||
nextSteps: [
|
||||
'klo demo',
|
||||
`klo setup --project-dir ${tempDir}`,
|
||||
'klo ingest <connection>',
|
||||
`klo agent sl list --json --query "gross revenue" --project-dir ${tempDir}`,
|
||||
],
|
||||
},
|
||||
});
|
||||
expect(io.stdout()).toBe('');
|
||||
});
|
||||
|
||||
it('prints guided JSON when semantic-layer search has no configured connections', async () => {
|
||||
const io = makeIo();
|
||||
|
||||
await expect(
|
||||
runKloAgent(
|
||||
{ command: 'sl-list', projectDir: tempDir, json: true, query: 'revenue' },
|
||||
io.io,
|
||||
{ createRuntime: async () => runtimeWithoutConnections() },
|
||||
),
|
||||
).resolves.toBe(1);
|
||||
|
||||
expect(JSON.parse(io.stderr())).toMatchObject({
|
||||
ok: false,
|
||||
error: {
|
||||
code: 'agent_sl_search_no_connections',
|
||||
message: `Semantic-layer search found no configured connections in ${tempDir}.`,
|
||||
nextSteps: [
|
||||
'klo demo',
|
||||
`klo setup --project-dir ${tempDir}`,
|
||||
'klo ingest <connection>',
|
||||
`klo agent sl list --json --query "revenue" --project-dir ${tempDir}`,
|
||||
],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('prints guided JSON when semantic-layer search asks for an unknown connection', async () => {
|
||||
const io = makeIo();
|
||||
|
||||
await expect(
|
||||
runKloAgent(
|
||||
{ command: 'sl-list', projectDir: tempDir, json: true, connectionId: 'missing', query: 'revenue' },
|
||||
io.io,
|
||||
{ createRuntime: async () => runtime() },
|
||||
),
|
||||
).resolves.toBe(1);
|
||||
|
||||
expect(JSON.parse(io.stderr())).toMatchObject({
|
||||
ok: false,
|
||||
error: {
|
||||
code: 'agent_sl_search_unknown_connection',
|
||||
message: `Semantic-layer search connection "missing" is not configured in ${tempDir}.`,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('prints guided JSON when semantic-layer search has no indexed sources', async () => {
|
||||
const fakeRuntime = runtime();
|
||||
const semanticLayer = fakeRuntime.ports.semanticLayer!;
|
||||
fakeRuntime.ports.semanticLayer = {
|
||||
...semanticLayer,
|
||||
listSources: vi.fn(async () => ({ sources: [], totalSources: 0 })),
|
||||
};
|
||||
const io = makeIo();
|
||||
|
||||
await expect(
|
||||
runKloAgent(
|
||||
{ command: 'sl-list', projectDir: tempDir, json: true, connectionId: 'warehouse', query: 'revenue' },
|
||||
io.io,
|
||||
{ createRuntime: async () => fakeRuntime },
|
||||
),
|
||||
).resolves.toBe(1);
|
||||
|
||||
expect(JSON.parse(io.stderr())).toMatchObject({
|
||||
ok: false,
|
||||
error: {
|
||||
code: 'agent_sl_search_no_indexed_sources',
|
||||
message: `Semantic-layer search found no indexed semantic-layer sources in ${tempDir}.`,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('returns JSON errors when required ports or records are missing', async () => {
|
||||
const io = makeIo();
|
||||
|
||||
await expect(
|
||||
runKloAgent({ command: 'wiki-read', projectDir: tempDir, json: true, pageId: 'missing' }, io.io, {
|
||||
createRuntime: async () =>
|
||||
runtime({
|
||||
ports: { knowledge: { read: vi.fn(async () => null) } },
|
||||
}) as never,
|
||||
}),
|
||||
).resolves.toBe(1);
|
||||
|
||||
expect(JSON.parse(io.stderr())).toMatchObject({
|
||||
ok: false,
|
||||
error: { message: expect.stringContaining('missing') },
|
||||
});
|
||||
});
|
||||
});
|
||||
214
packages/cli/src/agent.ts
Normal file
214
packages/cli/src/agent.ts
Normal file
|
|
@ -0,0 +1,214 @@
|
|||
import { readFile } from 'node:fs/promises';
|
||||
import type { KloCliIo } from './cli-runtime.js';
|
||||
import {
|
||||
createKloAgentRuntime,
|
||||
parseAgentMaxRows,
|
||||
readAgentJsonFile,
|
||||
writeAgentJson,
|
||||
writeAgentJsonError,
|
||||
type KloAgentRuntime,
|
||||
type KloAgentRuntimeDeps,
|
||||
} from './agent-runtime.js';
|
||||
import {
|
||||
isMissingProjectConfigError,
|
||||
missingConnectionSlSearchReadiness,
|
||||
missingProjectSlSearchReadiness,
|
||||
noConnectionsSlSearchReadiness,
|
||||
noIndexedSourcesSlSearchReadiness,
|
||||
type KloAgentSlSearchReadinessDetail,
|
||||
} from './agent-search-readiness.js';
|
||||
import { readKloSetupStatus, type KloSetupStatus } from './setup.js';
|
||||
|
||||
export type KloAgentArgs =
|
||||
| { command: 'tools'; projectDir: string; json: true }
|
||||
| { command: 'context'; projectDir: string; json: true }
|
||||
| { command: 'sl-list'; projectDir: string; json: true; connectionId?: string; query?: string }
|
||||
| { command: 'sl-read'; projectDir: string; json: true; connectionId?: string; sourceName: string }
|
||||
| {
|
||||
command: 'sl-query';
|
||||
projectDir: string;
|
||||
json: true;
|
||||
connectionId: string;
|
||||
queryFile: string;
|
||||
execute: boolean;
|
||||
maxRows?: number;
|
||||
}
|
||||
| { command: 'wiki-search'; projectDir: string; json: true; query: string; limit: number }
|
||||
| { command: 'wiki-read'; projectDir: string; json: true; pageId: string }
|
||||
| { command: 'sql-execute'; projectDir: string; json: true; connectionId: string; sqlFile: string; maxRows?: number };
|
||||
|
||||
export interface KloAgentDeps extends KloAgentRuntimeDeps {
|
||||
createRuntime?: (options: {
|
||||
projectDir: string;
|
||||
enableSemanticCompute: boolean;
|
||||
enableQueryExecution: boolean;
|
||||
}) => Promise<KloAgentRuntime>;
|
||||
readSetupStatus?: (
|
||||
projectDir: string,
|
||||
) => Promise<KloSetupStatus | { project: { path?: string; ready: boolean }; agents: unknown[] }>;
|
||||
}
|
||||
|
||||
const AGENT_TOOLS = [
|
||||
{ name: 'context', command: 'klo agent context --json' },
|
||||
{ name: 'sl.list', command: 'klo agent sl list --json [--connection-id <id>] [--query <text>]' },
|
||||
{ name: 'sl.read', command: 'klo agent sl read <sourceName> --json [--connection-id <id>]' },
|
||||
{
|
||||
name: 'sl.query',
|
||||
command: 'klo agent sl query --json --connection-id <id> --query-file <path> --execute --max-rows 100',
|
||||
},
|
||||
{ name: 'wiki.search', command: 'klo agent wiki search <query> --json [--limit 10]' },
|
||||
{ name: 'wiki.read', command: 'klo agent wiki read <pageId> --json' },
|
||||
{
|
||||
name: 'sql.execute',
|
||||
command: 'klo agent sql execute --json --connection-id <id> --sql-file <path> --max-rows 100',
|
||||
},
|
||||
] as const;
|
||||
|
||||
function writeAgentSlSearchReadinessError(io: KloCliIo, detail: KloAgentSlSearchReadinessDetail): void {
|
||||
writeAgentJsonError(io, detail.message, { code: detail.code, nextSteps: detail.nextSteps });
|
||||
}
|
||||
|
||||
async function runtimeFor(args: KloAgentArgs, deps: KloAgentDeps): Promise<KloAgentRuntime> {
|
||||
const needsSemanticCompute = args.command === 'sl-query';
|
||||
const needsQueryExecution = args.command === 'sql-execute' || (args.command === 'sl-query' && args.execute);
|
||||
return deps.createRuntime
|
||||
? deps.createRuntime({
|
||||
projectDir: args.projectDir,
|
||||
enableSemanticCompute: needsSemanticCompute,
|
||||
enableQueryExecution: needsQueryExecution,
|
||||
})
|
||||
: createKloAgentRuntime(
|
||||
{
|
||||
projectDir: args.projectDir,
|
||||
enableSemanticCompute: needsSemanticCompute,
|
||||
enableQueryExecution: needsQueryExecution,
|
||||
},
|
||||
deps,
|
||||
);
|
||||
}
|
||||
|
||||
function connectionIdForSource(runtime: KloAgentRuntime, requested: string | undefined): string {
|
||||
if (requested) return requested;
|
||||
const ids = Object.keys(runtime.project.config.connections ?? {});
|
||||
if (ids.length === 1) return ids[0] as string;
|
||||
throw new Error('Use --connection-id when the project has zero or multiple connections.');
|
||||
}
|
||||
|
||||
export async function runKloAgent(args: KloAgentArgs, io: KloCliIo, deps: KloAgentDeps = {}): Promise<number> {
|
||||
try {
|
||||
if (args.command === 'tools') {
|
||||
writeAgentJson(io, { projectDir: args.projectDir, tools: AGENT_TOOLS });
|
||||
return 0;
|
||||
}
|
||||
|
||||
const runtime = await runtimeFor(args, deps);
|
||||
|
||||
if (args.command === 'context') {
|
||||
const [status, connections, semanticLayer] = await Promise.all([
|
||||
(deps.readSetupStatus ?? readKloSetupStatus)(args.projectDir),
|
||||
runtime.ports.connections?.list() ?? [],
|
||||
runtime.ports.semanticLayer?.listSources({}) ?? { sources: [], totalSources: 0 },
|
||||
]);
|
||||
writeAgentJson(io, { projectDir: args.projectDir, status, connections, semanticLayer, tools: AGENT_TOOLS });
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (args.command === 'sl-list') {
|
||||
const semanticLayer = runtime.ports.semanticLayer;
|
||||
if (!semanticLayer) throw new Error('Semantic-layer tools are not available for this project.');
|
||||
if (args.query) {
|
||||
const connectionIds = Object.keys(runtime.project.config.connections ?? {});
|
||||
if (args.connectionId && !runtime.project.config.connections[args.connectionId]) {
|
||||
writeAgentSlSearchReadinessError(
|
||||
io,
|
||||
missingConnectionSlSearchReadiness(args.projectDir, args.connectionId, args.query),
|
||||
);
|
||||
return 1;
|
||||
}
|
||||
if (connectionIds.length === 0) {
|
||||
writeAgentSlSearchReadinessError(io, noConnectionsSlSearchReadiness(args.projectDir, args.query));
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
const listed = await semanticLayer.listSources({ connectionId: args.connectionId, query: args.query });
|
||||
if (args.query && listed.sources.length === 0) {
|
||||
const allSources = await semanticLayer.listSources({ connectionId: args.connectionId });
|
||||
if (allSources.totalSources === 0) {
|
||||
writeAgentSlSearchReadinessError(io, noIndexedSourcesSlSearchReadiness(args.projectDir, args.query));
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
writeAgentJson(io, listed);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (args.command === 'sl-read') {
|
||||
const semanticLayer = runtime.ports.semanticLayer;
|
||||
if (!semanticLayer) throw new Error('Semantic-layer tools are not available for this project.');
|
||||
const source = await semanticLayer.readSource({
|
||||
connectionId: connectionIdForSource(runtime, args.connectionId),
|
||||
sourceName: args.sourceName,
|
||||
});
|
||||
if (!source) throw new Error(`Semantic-layer source "${args.sourceName}" was not found.`);
|
||||
writeAgentJson(io, source);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (args.command === 'sl-query') {
|
||||
const semanticLayer = runtime.ports.semanticLayer;
|
||||
if (!semanticLayer) throw new Error('Semantic-layer tools are not available for this project.');
|
||||
const query = await readAgentJsonFile(args.queryFile);
|
||||
const maxRows = args.execute ? parseAgentMaxRows(args.maxRows) : args.maxRows;
|
||||
writeAgentJson(
|
||||
io,
|
||||
await semanticLayer.query({
|
||||
connectionId: args.connectionId,
|
||||
query: { ...query, ...(maxRows !== undefined ? { limit: maxRows } : {}) } as never,
|
||||
}),
|
||||
);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (args.command === 'wiki-search') {
|
||||
const knowledge = runtime.ports.knowledge;
|
||||
if (!knowledge) throw new Error('Wiki tools are not available for this project.');
|
||||
writeAgentJson(io, await knowledge.search({ userId: 'agent', query: args.query, limit: args.limit }));
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (args.command === 'wiki-read') {
|
||||
const knowledge = runtime.ports.knowledge;
|
||||
if (!knowledge) throw new Error('Wiki tools are not available for this project.');
|
||||
const page = await knowledge.read({ userId: 'agent', key: args.pageId });
|
||||
if (!page) throw new Error(`Wiki page "${args.pageId}" was not found.`);
|
||||
writeAgentJson(io, page);
|
||||
return 0;
|
||||
}
|
||||
|
||||
const queryExecutor = runtime.queryExecutor;
|
||||
if (!queryExecutor) throw new Error('SQL execution is not available for this project.');
|
||||
const connection = runtime.project.config.connections[args.connectionId];
|
||||
if (!connection) throw new Error(`Connection "${args.connectionId}" was not found.`);
|
||||
const maxRows = parseAgentMaxRows(args.maxRows);
|
||||
writeAgentJson(
|
||||
io,
|
||||
await queryExecutor.execute({
|
||||
connectionId: args.connectionId,
|
||||
projectDir: runtime.project.projectDir,
|
||||
connection,
|
||||
sql: await readFile(args.sqlFile, 'utf-8'),
|
||||
maxRows,
|
||||
}),
|
||||
);
|
||||
return 0;
|
||||
} catch (error) {
|
||||
if (args.command === 'sl-list' && args.query && isMissingProjectConfigError(error)) {
|
||||
writeAgentSlSearchReadinessError(io, missingProjectSlSearchReadiness(args.projectDir, args.query));
|
||||
return 1;
|
||||
}
|
||||
writeAgentJsonError(io, error instanceof Error ? error.message : String(error));
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
9
packages/cli/src/bin.ts
Normal file
9
packages/cli/src/bin.ts
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
import { installStartupProfileReporter, profileMark, profileSpan } from './startup-profile.js';
|
||||
|
||||
installStartupProfileReporter();
|
||||
profileMark('bin:entry');
|
||||
const { runKloCli } = await profileSpan('import ./cli-runtime.js', () => import('./cli-runtime.js'));
|
||||
profileMark('bin:runKloCli');
|
||||
process.exitCode = await runKloCli(process.argv.slice(2));
|
||||
11
packages/cli/src/clack.ts
Normal file
11
packages/cli/src/clack.ts
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
import { spinner } from '@clack/prompts';
|
||||
|
||||
export interface KloCliSpinner {
|
||||
start(message: string): void;
|
||||
stop(message: string): void;
|
||||
error(message: string): void;
|
||||
}
|
||||
|
||||
export function createClackSpinner(): KloCliSpinner {
|
||||
return spinner();
|
||||
}
|
||||
268
packages/cli/src/cli-program.ts
Normal file
268
packages/cli/src/cli-program.ts
Normal file
|
|
@ -0,0 +1,268 @@
|
|||
import { Command, InvalidArgumentError } from '@commander-js/extra-typings';
|
||||
import type { KloCliDeps, KloCliIo, KloCliPackageInfo } from './cli-runtime.js';
|
||||
import { registerAgentCommands } from './commands/agent-commands.js';
|
||||
import { registerConnectionCommands } from './commands/connection-commands.js';
|
||||
import { registerWikiCommands } from './commands/knowledge-commands.js';
|
||||
import { registerPublicIngestCommands } from './commands/public-ingest-commands.js';
|
||||
import { registerServeCommands } from './commands/serve-commands.js';
|
||||
import { registerSetupCommands } from './commands/setup-commands.js';
|
||||
import { registerSlCommands } from './commands/sl-commands.js';
|
||||
import { registerStatusCommands } from './commands/status-commands.js';
|
||||
import { registerDevCommands } from './dev.js';
|
||||
import { findNearestKloProjectDir, resolveKloProjectDir } from './project-resolver.js';
|
||||
import { profileMark, profileSpan } from './startup-profile.js';
|
||||
|
||||
profileMark('module:cli-program');
|
||||
|
||||
export interface KloCliCommandContext {
|
||||
io: KloCliIo;
|
||||
deps: KloCliDeps;
|
||||
setExitCode: (code: number) => void;
|
||||
runInit: (args: { projectDir: string; projectName?: string; force: boolean }, io: KloCliIo) => Promise<number>;
|
||||
writeDebug?: (command: string, commandContext: CommandWithGlobalOptions) => void;
|
||||
}
|
||||
|
||||
export interface OutputModeOptions {
|
||||
plain?: boolean;
|
||||
json?: boolean;
|
||||
viz?: boolean;
|
||||
input?: boolean;
|
||||
}
|
||||
|
||||
interface KloCommanderProgramOptions {
|
||||
runInit: (args: { projectDir: string; projectName?: string; force: boolean }, io: KloCliIo) => Promise<number>;
|
||||
}
|
||||
|
||||
type CommanderExitLike = { exitCode: number; code: string; message: string };
|
||||
|
||||
interface KloGlobalOptionValues {
|
||||
projectDir?: string;
|
||||
debug?: boolean;
|
||||
}
|
||||
|
||||
export interface CommandWithGlobalOptions {
|
||||
opts: () => object;
|
||||
optsWithGlobals?: () => object;
|
||||
}
|
||||
|
||||
function isCommanderExit(error: unknown): error is CommanderExitLike {
|
||||
return (
|
||||
typeof error === 'object' &&
|
||||
error !== null &&
|
||||
'exitCode' in error &&
|
||||
typeof (error as { exitCode: unknown }).exitCode === 'number' &&
|
||||
'code' in error &&
|
||||
typeof (error as { code: unknown }).code === 'string'
|
||||
);
|
||||
}
|
||||
|
||||
export function collectOption(value: string, previous: string[] = []): string[] {
|
||||
return [...previous, value];
|
||||
}
|
||||
|
||||
export function parsePositiveIntegerOption(value: string): number {
|
||||
const parsed = Number(value);
|
||||
if (!Number.isInteger(parsed) || parsed < 1) {
|
||||
throw new InvalidArgumentError('must be a positive integer');
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
export function parseNonNegativeIntegerOption(value: string): number {
|
||||
const parsed = Number(value);
|
||||
if (!Number.isInteger(parsed) || parsed < 0) {
|
||||
throw new InvalidArgumentError('must be a non-negative integer');
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
export function parseBooleanStringOption(value: string): boolean {
|
||||
if (value === 'true') {
|
||||
return true;
|
||||
}
|
||||
if (value === 'false') {
|
||||
return false;
|
||||
}
|
||||
throw new InvalidArgumentError('must be true or false');
|
||||
}
|
||||
|
||||
export function parseSafeConnectionIdOption(value: string): string {
|
||||
if (!/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(value)) {
|
||||
throw new InvalidArgumentError(`Unsafe connection id: ${value}`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
export function parseNonEmptyAssignmentOption(value: string): { key: string; value: string } {
|
||||
const separatorIndex = value.indexOf('=');
|
||||
if (separatorIndex <= 0 || separatorIndex === value.length - 1) {
|
||||
throw new InvalidArgumentError('must be a non-empty <key>=<value> assignment');
|
||||
}
|
||||
return {
|
||||
key: value.slice(0, separatorIndex),
|
||||
value: value.slice(separatorIndex + 1),
|
||||
};
|
||||
}
|
||||
|
||||
function optionsWithGlobals(command: CommandWithGlobalOptions): KloGlobalOptionValues {
|
||||
const options = command.optsWithGlobals ? command.optsWithGlobals() : command.opts();
|
||||
const values = options as { projectDir?: unknown; debug?: unknown };
|
||||
return {
|
||||
projectDir: typeof values.projectDir === 'string' ? values.projectDir : undefined,
|
||||
debug: typeof values.debug === 'boolean' ? values.debug : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveCommandProjectDir(command: CommandWithGlobalOptions): string {
|
||||
return resolveKloProjectDir({ explicitProjectDir: optionsWithGlobals(command).projectDir });
|
||||
}
|
||||
|
||||
export function resolveCommandProjectDirOverride(command: CommandWithGlobalOptions): string | undefined {
|
||||
return optionsWithGlobals(command).projectDir ?? process.env.KLO_PROJECT_DIR;
|
||||
}
|
||||
|
||||
function createBaseProgram(info: KloCliPackageInfo, io: KloCliIo): Command {
|
||||
return new Command()
|
||||
.name('klo')
|
||||
.description('Standalone KLO developer CLI')
|
||||
.option('--project-dir <path>', 'KLO project directory (default: KLO_PROJECT_DIR, nearest klo.yaml, or cwd)')
|
||||
.option('--debug', 'Enable diagnostic logging to stderr')
|
||||
.version(`${info.name} ${info.version}`, '-v, --version', 'Show CLI version')
|
||||
.helpOption('-h, --help', 'Show this help text')
|
||||
.configureHelp({ showGlobalOptions: true })
|
||||
.addHelpText(
|
||||
'after',
|
||||
'\nAdvanced:\n klo dev Low-level diagnostics, scans, adapter commands, and mapping tools.\n',
|
||||
)
|
||||
.showHelpAfterError()
|
||||
.exitOverride()
|
||||
.configureOutput({
|
||||
writeOut: (chunk) => io.stdout.write(chunk),
|
||||
writeErr: (chunk) => io.stderr.write(chunk),
|
||||
outputError: (chunk, write) => write(chunk),
|
||||
});
|
||||
}
|
||||
|
||||
function writeDebug(io: KloCliIo, commandContext: CommandWithGlobalOptions, command: string): void {
|
||||
const global = optionsWithGlobals(commandContext);
|
||||
if (global.debug !== true) {
|
||||
return;
|
||||
}
|
||||
io.stderr.write(`[debug] projectDir=${resolveCommandProjectDir(commandContext)}\n`);
|
||||
io.stderr.write(`[debug] dispatch=${command}\n`);
|
||||
}
|
||||
|
||||
function formatCliError(error: unknown): string {
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
|
||||
async function runBareInteractiveCommand(
|
||||
program: Command,
|
||||
io: KloCliIo,
|
||||
context: KloCliCommandContext,
|
||||
): Promise<number> {
|
||||
const nearestProjectDir = findNearestKloProjectDir(process.cwd());
|
||||
const envProjectDir = process.env.KLO_PROJECT_DIR;
|
||||
const runner = context.deps.setup ?? (await import('./setup.js')).runKloSetup;
|
||||
|
||||
if (!nearestProjectDir && !envProjectDir) {
|
||||
return await runner(
|
||||
{
|
||||
command: 'run',
|
||||
projectDir: resolveKloProjectDir(),
|
||||
mode: 'auto',
|
||||
agents: false,
|
||||
agentScope: 'project',
|
||||
agentInstallMode: 'cli',
|
||||
skipAgents: false,
|
||||
inputMode: 'auto',
|
||||
yes: false,
|
||||
skipLlm: false,
|
||||
skipEmbeddings: false,
|
||||
databaseSchemas: [],
|
||||
skipDatabases: false,
|
||||
skipSources: false,
|
||||
},
|
||||
io,
|
||||
);
|
||||
}
|
||||
|
||||
program.outputHelp();
|
||||
return 0;
|
||||
}
|
||||
|
||||
export async function runCommanderKloCli(
|
||||
argv: string[],
|
||||
io: KloCliIo,
|
||||
deps: KloCliDeps,
|
||||
info: KloCliPackageInfo,
|
||||
options: KloCommanderProgramOptions,
|
||||
): Promise<number> {
|
||||
profileMark('commander:entry');
|
||||
let exitCode = 0;
|
||||
const program = createBaseProgram(info, io);
|
||||
profileMark('commander:base-program');
|
||||
const context: KloCliCommandContext = {
|
||||
io,
|
||||
deps,
|
||||
setExitCode: (code: number) => {
|
||||
exitCode = code;
|
||||
},
|
||||
runInit: options.runInit,
|
||||
writeDebug: (command: string, commandContext: CommandWithGlobalOptions) => {
|
||||
writeDebug(io, commandContext, command);
|
||||
},
|
||||
};
|
||||
|
||||
registerSetupCommands(program, context);
|
||||
profileMark('commander:register-setup');
|
||||
|
||||
registerConnectionCommands(program, context);
|
||||
profileMark('commander:register-connection');
|
||||
|
||||
registerPublicIngestCommands(program, context);
|
||||
profileMark('commander:register-public-ingest');
|
||||
|
||||
registerWikiCommands(program, context);
|
||||
profileMark('commander:register-wiki');
|
||||
|
||||
registerSlCommands(program, context);
|
||||
profileMark('commander:register-sl');
|
||||
|
||||
registerServeCommands(program, context);
|
||||
profileMark('commander:register-serve');
|
||||
|
||||
registerStatusCommands(program, context);
|
||||
profileMark('commander:register-status');
|
||||
|
||||
registerAgentCommands(program, context);
|
||||
profileMark('commander:register-agent');
|
||||
|
||||
registerDevCommands(program, context);
|
||||
profileMark('commander:register-dev');
|
||||
|
||||
if (argv.length === 0) {
|
||||
if (io.stdout.isTTY === true) {
|
||||
try {
|
||||
return await runBareInteractiveCommand(program, io, context);
|
||||
} catch (error) {
|
||||
io.stderr.write(`${formatCliError(error)}\n`);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
program.outputHelp();
|
||||
return 0;
|
||||
}
|
||||
|
||||
try {
|
||||
await profileSpan('commander:parseAsync', () => program.parseAsync(argv, { from: 'user' }));
|
||||
} catch (error) {
|
||||
if (isCommanderExit(error)) {
|
||||
return error.exitCode === 0 ? 0 : 1;
|
||||
}
|
||||
io.stderr.write(`${formatCliError(error)}\n`);
|
||||
return 1;
|
||||
}
|
||||
|
||||
return exitCode;
|
||||
}
|
||||
89
packages/cli/src/cli-runtime.ts
Normal file
89
packages/cli/src/cli-runtime.ts
Normal file
|
|
@ -0,0 +1,89 @@
|
|||
import type { KloConnectionMetabaseSetupArgs } from './commands/connection-metabase-setup.js';
|
||||
import type { KloConnectionNotionArgs } from './commands/connection-notion.js';
|
||||
import type { KloAgentArgs } from './agent.js';
|
||||
import type { KloConnectionArgs } from './connection.js';
|
||||
import type { KloDemoArgs } from './demo.js';
|
||||
import type { KloDoctorArgs } from './doctor.js';
|
||||
import type { KloIngestArgs } from './ingest.js';
|
||||
import type { KloKnowledgeArgs } from './knowledge.js';
|
||||
import type { KloPublicIngestArgs } from './public-ingest.js';
|
||||
import type { KloScanArgs } from './scan.js';
|
||||
import type { KloServeArgs } from './serve.js';
|
||||
import type { KloSetupArgs } from './setup.js';
|
||||
import type { KloSlArgs } from './sl.js';
|
||||
import { profileMark, profileSpan } from './startup-profile.js';
|
||||
|
||||
profileMark('module:cli-runtime');
|
||||
|
||||
export interface KloCliPackageInfo {
|
||||
name: '@klo/cli';
|
||||
version: '0.0.0-private';
|
||||
contextPackageName: '@klo/context';
|
||||
}
|
||||
|
||||
export interface KloCliIo {
|
||||
stdout: { isTTY?: boolean; write(chunk: string): void };
|
||||
stderr: { write(chunk: string): void };
|
||||
}
|
||||
|
||||
export interface KloCliDeps {
|
||||
serveStdio?: (args: KloServeArgs) => Promise<number>;
|
||||
setup?: (args: KloSetupArgs, io: KloCliIo) => Promise<number>;
|
||||
agent?: (args: KloAgentArgs, io: KloCliIo) => Promise<number>;
|
||||
connection?: (args: KloConnectionArgs, io: KloCliIo) => Promise<number>;
|
||||
connectionNotion?: (args: KloConnectionNotionArgs, io: KloCliIo) => Promise<number>;
|
||||
connectionMetabaseSetup?: (args: KloConnectionMetabaseSetupArgs, io: KloCliIo) => Promise<number>;
|
||||
demo?: (args: KloDemoArgs, io: KloCliIo) => Promise<number>;
|
||||
doctor?: (args: KloDoctorArgs, io: KloCliIo) => Promise<number>;
|
||||
ingest?: (args: KloIngestArgs, io: KloCliIo) => Promise<number>;
|
||||
publicIngest?: (args: KloPublicIngestArgs, io: KloCliIo) => Promise<number>;
|
||||
scan?: (args: KloScanArgs, io: KloCliIo) => Promise<number>;
|
||||
knowledge?: (args: KloKnowledgeArgs, io: KloCliIo) => Promise<number>;
|
||||
sl?: (args: KloSlArgs, io: KloCliIo) => Promise<number>;
|
||||
}
|
||||
|
||||
export function getKloCliPackageInfo(): KloCliPackageInfo {
|
||||
return {
|
||||
name: '@klo/cli',
|
||||
version: '0.0.0-private',
|
||||
contextPackageName: '@klo/context',
|
||||
};
|
||||
}
|
||||
|
||||
async function runInit(
|
||||
args: { projectDir: string; projectName?: string; force: boolean },
|
||||
io: KloCliIo,
|
||||
): Promise<number> {
|
||||
const { initKloProject } = await import('@klo/context/project');
|
||||
const result = await initKloProject({
|
||||
projectDir: args.projectDir,
|
||||
projectName: args.projectName,
|
||||
force: args.force,
|
||||
});
|
||||
|
||||
io.stdout.write(`Initialized KLO project at ${result.projectDir}\n`);
|
||||
io.stdout.write(`Config: ${result.configPath}\n`);
|
||||
io.stdout.write(`Commit: ${result.commitHash ?? 'none'}\n`);
|
||||
return 0;
|
||||
}
|
||||
|
||||
export async function runInitForCommander(
|
||||
args: { projectDir: string; projectName?: string; force: boolean },
|
||||
io: KloCliIo,
|
||||
): Promise<number> {
|
||||
return await runInit(args, io);
|
||||
}
|
||||
|
||||
export async function runKloCli(
|
||||
argv = process.argv.slice(2),
|
||||
io: KloCliIo = process,
|
||||
deps: KloCliDeps = {},
|
||||
): Promise<number> {
|
||||
const info = getKloCliPackageInfo();
|
||||
profileMark('runtime:runKloCli');
|
||||
const { runCommanderKloCli } = await profileSpan('import ./cli-program.js', () => import('./cli-program.js'));
|
||||
|
||||
return await runCommanderKloCli(argv, io, deps, info, {
|
||||
runInit: runInitForCommander,
|
||||
});
|
||||
}
|
||||
85
packages/cli/src/command-schemas.ts
Normal file
85
packages/cli/src/command-schemas.ts
Normal file
|
|
@ -0,0 +1,85 @@
|
|||
import { z } from 'zod';
|
||||
|
||||
const projectDirSchema = z.string().min(1);
|
||||
const safeConnectionIdSchema = z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/, 'Unsafe connection id');
|
||||
const stringArraySchema = z.array(z.string());
|
||||
|
||||
export const connectionAddCommandSchema = z.object({
|
||||
command: z.literal('add'),
|
||||
projectDir: projectDirSchema,
|
||||
driver: z.string().min(1),
|
||||
connectionId: safeConnectionIdSchema,
|
||||
url: z.string().optional(),
|
||||
schemas: stringArraySchema,
|
||||
readonly: z.boolean(),
|
||||
force: z.boolean(),
|
||||
allowLiteralCredentials: z.boolean(),
|
||||
notion: z
|
||||
.object({
|
||||
authTokenRef: z.string().min(1),
|
||||
crawlMode: z.enum(['all_accessible', 'selected_roots']),
|
||||
rootPageIds: stringArraySchema,
|
||||
rootDatabaseIds: stringArraySchema,
|
||||
rootDataSourceIds: stringArraySchema,
|
||||
maxPagesPerRun: z.number().int().positive().optional(),
|
||||
maxKnowledgeCreatesPerRun: z.number().int().nonnegative().optional(),
|
||||
maxKnowledgeUpdatesPerRun: z.number().int().nonnegative().optional(),
|
||||
})
|
||||
.optional(),
|
||||
});
|
||||
|
||||
export const wikiWriteCommandSchema = z.object({
|
||||
command: z.literal('write'),
|
||||
projectDir: projectDirSchema,
|
||||
key: z.string().min(1),
|
||||
scope: z.enum(['GLOBAL', 'USER']),
|
||||
userId: z.string().min(1),
|
||||
summary: z.string().min(1),
|
||||
content: z.string().min(1),
|
||||
tags: stringArraySchema,
|
||||
refs: stringArraySchema,
|
||||
slRefs: stringArraySchema,
|
||||
});
|
||||
|
||||
const orderBySchema = z.union([
|
||||
z.string().min(1),
|
||||
z.object({
|
||||
field: z.string().min(1),
|
||||
direction: z.enum(['asc', 'desc']).optional(),
|
||||
}),
|
||||
]);
|
||||
|
||||
export const slQueryCommandSchema = z.object({
|
||||
command: z.literal('query'),
|
||||
projectDir: projectDirSchema,
|
||||
connectionId: z.string().min(1).optional(),
|
||||
query: z.object({
|
||||
measures: z.array(z.string().min(1)).min(1),
|
||||
dimensions: stringArraySchema,
|
||||
filters: stringArraySchema.optional(),
|
||||
segments: stringArraySchema.optional(),
|
||||
order_by: z.array(orderBySchema).optional(),
|
||||
limit: z.number().int().positive().optional(),
|
||||
include_empty: z.literal(true).optional(),
|
||||
}),
|
||||
format: z.enum(['json', 'sql']),
|
||||
execute: z.boolean(),
|
||||
maxRows: z.number().int().positive().optional(),
|
||||
});
|
||||
|
||||
export const publicIngestRunCommandSchema = z.object({
|
||||
command: z.literal('run'),
|
||||
projectDir: projectDirSchema,
|
||||
targetConnectionId: safeConnectionIdSchema.optional(),
|
||||
all: z.boolean(),
|
||||
json: z.boolean(),
|
||||
inputMode: z.enum(['auto', 'disabled']),
|
||||
});
|
||||
|
||||
export const publicIngestReadCommandSchema = z.object({
|
||||
command: z.enum(['status', 'watch']),
|
||||
projectDir: projectDirSchema,
|
||||
runId: z.string().min(1).optional(),
|
||||
json: z.boolean(),
|
||||
inputMode: z.enum(['auto', 'disabled']),
|
||||
});
|
||||
137
packages/cli/src/commands/agent-commands.ts
Normal file
137
packages/cli/src/commands/agent-commands.ts
Normal file
|
|
@ -0,0 +1,137 @@
|
|||
import { Option, type Command } from '@commander-js/extra-typings';
|
||||
import type { KloAgentArgs } from '../agent.js';
|
||||
import type { KloCliCommandContext } from '../cli-program.js';
|
||||
import { parsePositiveIntegerOption, resolveCommandProjectDir } from '../cli-program.js';
|
||||
|
||||
async function runAgent(context: KloCliCommandContext, args: KloAgentArgs): Promise<void> {
|
||||
const runner = context.deps.agent ?? (await import('../agent.js')).runKloAgent;
|
||||
context.setExitCode(await runner(args, context.io));
|
||||
}
|
||||
|
||||
function jsonOption(): Option {
|
||||
return new Option('--json', 'Print JSON output').makeOptionMandatory();
|
||||
}
|
||||
|
||||
export function registerAgentCommands(program: Command, context: KloCliCommandContext): void {
|
||||
const agent = program
|
||||
.command('agent', { hidden: true })
|
||||
.description('Machine-readable KLO commands for coding agents')
|
||||
.showHelpAfterError();
|
||||
|
||||
agent.hook('preAction', (_thisCommand, actionCommand) => {
|
||||
context.writeDebug?.('agent', actionCommand);
|
||||
});
|
||||
|
||||
agent
|
||||
.command('tools')
|
||||
.description('Print available agent-facing KLO tools')
|
||||
.addOption(jsonOption())
|
||||
.action(async (_options, command) => {
|
||||
await runAgent(context, { command: 'tools', projectDir: resolveCommandProjectDir(command), json: true });
|
||||
});
|
||||
|
||||
agent
|
||||
.command('context')
|
||||
.description('Print project context for agent planning')
|
||||
.addOption(jsonOption())
|
||||
.action(async (_options, command) => {
|
||||
await runAgent(context, { command: 'context', projectDir: resolveCommandProjectDir(command), json: true });
|
||||
});
|
||||
|
||||
const sl = agent.command('sl').description('Semantic-layer agent commands');
|
||||
sl.command('list')
|
||||
.description('List semantic-layer sources')
|
||||
.addOption(jsonOption())
|
||||
.option('--connection-id <id>', 'Filter by connection id')
|
||||
.option('--query <text>', 'Search source names and descriptions')
|
||||
.action(async (options: { connectionId?: string; query?: string }, command) => {
|
||||
await runAgent(context, {
|
||||
command: 'sl-list',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
json: true,
|
||||
...(options.connectionId ? { connectionId: options.connectionId } : {}),
|
||||
...(options.query ? { query: options.query } : {}),
|
||||
});
|
||||
});
|
||||
sl.command('read')
|
||||
.description('Read one semantic-layer source')
|
||||
.argument('<sourceName>')
|
||||
.addOption(jsonOption())
|
||||
.option('--connection-id <id>', 'Connection id containing the source')
|
||||
.action(async (sourceName: string, options: { connectionId?: string }, command) => {
|
||||
await runAgent(context, {
|
||||
command: 'sl-read',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
json: true,
|
||||
sourceName,
|
||||
...(options.connectionId ? { connectionId: options.connectionId } : {}),
|
||||
});
|
||||
});
|
||||
sl.command('query')
|
||||
.description('Run a semantic-layer query JSON file')
|
||||
.addOption(jsonOption())
|
||||
.requiredOption('--connection-id <id>', 'Connection id for execution')
|
||||
.requiredOption('--query-file <path>', 'JSON semantic-layer query file')
|
||||
.option('--execute', 'Execute the compiled query against the connection', false)
|
||||
.option('--max-rows <number>', 'Maximum rows to return when executing', parsePositiveIntegerOption)
|
||||
.action(
|
||||
async (
|
||||
options: { connectionId: string; queryFile: string; execute: boolean; maxRows?: number },
|
||||
command,
|
||||
) => {
|
||||
await runAgent(context, {
|
||||
command: 'sl-query',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
json: true,
|
||||
connectionId: options.connectionId,
|
||||
queryFile: options.queryFile,
|
||||
execute: options.execute,
|
||||
...(options.maxRows !== undefined ? { maxRows: options.maxRows } : {}),
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
const wiki = agent.command('wiki').description('KLO wiki agent commands');
|
||||
wiki
|
||||
.command('search')
|
||||
.description('Search KLO wiki pages')
|
||||
.argument('<query>')
|
||||
.addOption(jsonOption())
|
||||
.option('--limit <number>', 'Maximum search results', parsePositiveIntegerOption, 10)
|
||||
.action(async (query: string, options: { limit: number }, command) => {
|
||||
await runAgent(context, {
|
||||
command: 'wiki-search',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
json: true,
|
||||
query,
|
||||
limit: options.limit,
|
||||
});
|
||||
});
|
||||
wiki
|
||||
.command('read')
|
||||
.description('Read one KLO wiki page')
|
||||
.argument('<pageId>')
|
||||
.addOption(jsonOption())
|
||||
.action(async (pageId: string, _options, command) => {
|
||||
await runAgent(context, { command: 'wiki-read', projectDir: resolveCommandProjectDir(command), json: true, pageId });
|
||||
});
|
||||
|
||||
const sql = agent.command('sql').description('Safe SQL execution commands');
|
||||
sql
|
||||
.command('execute')
|
||||
.description('Execute read-only SQL with a row limit')
|
||||
.addOption(jsonOption())
|
||||
.requiredOption('--connection-id <id>', 'Connection id for execution')
|
||||
.requiredOption('--sql-file <path>', 'SQL file to execute')
|
||||
.requiredOption('--max-rows <number>', 'Maximum rows to return', parsePositiveIntegerOption)
|
||||
.action(async (options: { connectionId: string; sqlFile: string; maxRows: number }, command) => {
|
||||
await runAgent(context, {
|
||||
command: 'sql-execute',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
json: true,
|
||||
connectionId: options.connectionId,
|
||||
sqlFile: options.sqlFile,
|
||||
maxRows: options.maxRows,
|
||||
});
|
||||
});
|
||||
}
|
||||
47
packages/cli/src/commands/completion-commands.ts
Normal file
47
packages/cli/src/commands/completion-commands.ts
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
import type { CommandUnknownOpts } from '@commander-js/extra-typings';
|
||||
import type { KloCliCommandContext } from '../cli-program.js';
|
||||
import { completeCommanderInput, installZshCompletion, zshCompletionScript } from '../completion.js';
|
||||
|
||||
export function registerCompletionCommands(
|
||||
program: CommandUnknownOpts,
|
||||
context: KloCliCommandContext,
|
||||
completionRoot: CommandUnknownOpts = program,
|
||||
): void {
|
||||
program
|
||||
.command('completion')
|
||||
.description('Generate shell completion scripts')
|
||||
.command('zsh')
|
||||
.description('Generate zsh completion script')
|
||||
.option('--install', 'Install zsh completion into ~/.zfunc and update ~/.zshrc', false)
|
||||
.action(async (options: { install?: boolean }) => {
|
||||
if (options.install === true) {
|
||||
const result = await installZshCompletion();
|
||||
context.io.stdout.write(`Installed zsh completion: ${result.completionPath}\n`);
|
||||
context.io.stdout.write(`Updated zsh config: ${result.zshrcPath}\n`);
|
||||
context.io.stdout.write('Restart your shell or run: source ~/.zshrc\n');
|
||||
context.setExitCode(0);
|
||||
return;
|
||||
}
|
||||
context.io.stdout.write(zshCompletionScript());
|
||||
context.setExitCode(0);
|
||||
});
|
||||
|
||||
program
|
||||
.command('__complete', { hidden: true })
|
||||
.description('Internal shell completion endpoint')
|
||||
.requiredOption('--shell <shell>', 'Shell requesting completions')
|
||||
.requiredOption('--position <position>', 'Current shell word position', (value) => Number(value))
|
||||
.argument('[words...]', 'Current shell words')
|
||||
.allowUnknownOption()
|
||||
.allowExcessArguments()
|
||||
.action((words: string[], options: { shell: string; position: number }) => {
|
||||
if (options.shell !== 'zsh') {
|
||||
context.setExitCode(1);
|
||||
return;
|
||||
}
|
||||
for (const completion of completeCommanderInput(completionRoot, { position: options.position, words })) {
|
||||
context.io.stdout.write(`${completion}\n`);
|
||||
}
|
||||
context.setExitCode(0);
|
||||
});
|
||||
}
|
||||
346
packages/cli/src/commands/connection-commands.ts
Normal file
346
packages/cli/src/commands/connection-commands.ts
Normal file
|
|
@ -0,0 +1,346 @@
|
|||
import { type Command, InvalidArgumentError, Option } from '@commander-js/extra-typings';
|
||||
import {
|
||||
collectOption,
|
||||
type KloCliCommandContext,
|
||||
parseBooleanStringOption,
|
||||
parseNonEmptyAssignmentOption,
|
||||
parseNonNegativeIntegerOption,
|
||||
parsePositiveIntegerOption,
|
||||
parseSafeConnectionIdOption,
|
||||
resolveCommandProjectDir,
|
||||
} from '../cli-program.js';
|
||||
import { connectionAddCommandSchema } from '../command-schemas.js';
|
||||
import type { KloConnectionArgs } from '../connection.js';
|
||||
import { profileMark } from '../startup-profile.js';
|
||||
import type { KloConnectionMappingArgs } from './connection-mapping.js';
|
||||
import { registerConnectionMetabaseCommands } from './connection-metabase-commands.js';
|
||||
import { registerConnectionNotionCommands } from './connection-notion-commands.js';
|
||||
|
||||
profileMark('module:commands/connection-commands');
|
||||
|
||||
const CRAWL_MODE_CHOICES = ['all_accessible', 'selected_roots'] as const;
|
||||
const SYNC_MODE_CHOICES = ['ALL', 'ONLY', 'EXCEPT'] as const;
|
||||
|
||||
function parseCsvIds(value: string): number[] {
|
||||
return value
|
||||
.split(',')
|
||||
.filter(Boolean)
|
||||
.map((item) => parsePositiveIntegerOption(item));
|
||||
}
|
||||
|
||||
function parseCsvStrings(value: string): string[] {
|
||||
return value
|
||||
.split(',')
|
||||
.map((item) => item.trim())
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function parseMappingFieldOption(value: string): 'databaseMappings' | 'connectionMappings' {
|
||||
if (value === 'databaseMappings' || value === 'connectionMappings') {
|
||||
return value;
|
||||
}
|
||||
throw new InvalidArgumentError('must be databaseMappings or connectionMappings');
|
||||
}
|
||||
|
||||
async function runConnectionArgs(context: KloCliCommandContext, args: KloConnectionArgs): Promise<void> {
|
||||
const runner = context.deps.connection ?? (await import('../connection.js')).runKloConnection;
|
||||
context.setExitCode(await runner(args, context.io));
|
||||
}
|
||||
|
||||
async function runMappingArgs(context: KloCliCommandContext, args: KloConnectionMappingArgs): Promise<void> {
|
||||
const { runKloConnectionMapping } = await import('./connection-mapping.js');
|
||||
context.setExitCode(await runKloConnectionMapping(args, context.io));
|
||||
}
|
||||
|
||||
export function registerConnectionCommands(program: Command, context: KloCliCommandContext, commandName = 'connection'): void {
|
||||
const connection = program
|
||||
.command(commandName)
|
||||
.description('Add, list, test, and map data sources')
|
||||
.showHelpAfterError()
|
||||
.addHelpText(
|
||||
'after',
|
||||
'\nProject directory defaults to KLO_PROJECT_DIR when set, otherwise the nearest klo.yaml or current working directory.\n',
|
||||
);
|
||||
connection.hook('preAction', (_thisCommand, actionCommand) => {
|
||||
context.writeDebug?.(commandName, actionCommand);
|
||||
});
|
||||
|
||||
connection
|
||||
.command('list')
|
||||
.description('List configured connections')
|
||||
.action(async (_options: unknown, command) => {
|
||||
await runConnectionArgs(context, { command: 'list', projectDir: resolveCommandProjectDir(command) });
|
||||
});
|
||||
|
||||
connection
|
||||
.command('test')
|
||||
.description('Test a configured connection')
|
||||
.argument('<connectionId>', 'KLO connection id')
|
||||
.action(async (connectionId: string, _options: unknown, command) => {
|
||||
await runConnectionArgs(context, {
|
||||
command: 'test',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
connectionId,
|
||||
});
|
||||
});
|
||||
|
||||
connection
|
||||
.command('add')
|
||||
.description('Add or replace a configured connection')
|
||||
.argument('<driver>', 'Connection driver')
|
||||
.argument('<connectionId>', 'KLO connection id')
|
||||
.option('--url <url>', 'Connection URL, env:NAME, or file:/path reference')
|
||||
.option('--schema <schema>', 'Schema to include; repeatable', collectOption, [])
|
||||
.option('--readonly', 'Mark the connection as read-only', false)
|
||||
.option('--force', 'Replace an existing connection', false)
|
||||
.option('--allow-literal-credentials', 'Allow writing a literal credential URL to klo.yaml', false)
|
||||
.addOption(new Option('--token-env <name>', 'Environment variable containing Notion auth token').conflicts('tokenFile'))
|
||||
.addOption(new Option('--token-file <path>', 'File containing Notion auth token').conflicts('tokenEnv'))
|
||||
.addOption(
|
||||
new Option('--crawl-mode <mode>', 'Notion crawl mode: all_accessible or selected_roots')
|
||||
.choices(CRAWL_MODE_CHOICES)
|
||||
.default('selected_roots'),
|
||||
)
|
||||
.option('--root-page-id <id>', 'Root page to crawl; repeatable', collectOption, [])
|
||||
.option('--root-database-id <id>', 'Root database to crawl; repeatable', collectOption, [])
|
||||
.option('--root-data-source-id <id>', 'Root data source to crawl; repeatable', collectOption, [])
|
||||
.option('--max-pages <n>', 'Maximum pages per run', parsePositiveIntegerOption)
|
||||
.option('--max-knowledge-creates <n>', 'Maximum knowledge creates per run', parseNonNegativeIntegerOption)
|
||||
.option('--max-knowledge-updates <n>', 'Maximum knowledge updates per run', parseNonNegativeIntegerOption)
|
||||
.action(async (driver: string, connectionId: string, options, command) => {
|
||||
const notion =
|
||||
driver === 'notion'
|
||||
? {
|
||||
authTokenRef: options.tokenEnv
|
||||
? `env:${options.tokenEnv}`
|
||||
: options.tokenFile
|
||||
? `file:${options.tokenFile}`
|
||||
: '',
|
||||
crawlMode: options.crawlMode,
|
||||
rootPageIds: options.rootPageId,
|
||||
rootDatabaseIds: options.rootDatabaseId,
|
||||
rootDataSourceIds: options.rootDataSourceId,
|
||||
maxPagesPerRun: options.maxPages,
|
||||
maxKnowledgeCreatesPerRun: options.maxKnowledgeCreates,
|
||||
maxKnowledgeUpdatesPerRun: options.maxKnowledgeUpdates,
|
||||
}
|
||||
: undefined;
|
||||
|
||||
if (driver === 'notion' && !notion?.authTokenRef) {
|
||||
throw new Error('connection add notion requires --token-env NAME or --token-file PATH');
|
||||
}
|
||||
if (
|
||||
driver === 'notion' &&
|
||||
notion?.crawlMode === 'selected_roots' &&
|
||||
notion.rootPageIds.length + notion.rootDatabaseIds.length + notion.rootDataSourceIds.length === 0
|
||||
) {
|
||||
throw new Error('connection add notion selected_roots requires at least one root id');
|
||||
}
|
||||
|
||||
const args = connectionAddCommandSchema.parse({
|
||||
command: 'add',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
driver,
|
||||
connectionId,
|
||||
url: options.url,
|
||||
schemas: options.schema.filter(Boolean),
|
||||
readonly: options.readonly === true,
|
||||
force: options.force === true,
|
||||
allowLiteralCredentials: options.allowLiteralCredentials === true,
|
||||
notion,
|
||||
});
|
||||
|
||||
await runConnectionArgs(context, args);
|
||||
});
|
||||
|
||||
connection
|
||||
.command('remove')
|
||||
.description('Remove a configured connection from klo.yaml')
|
||||
.argument('<connectionId>', 'KLO connection id')
|
||||
.option('--force', 'Remove without prompting', false)
|
||||
.option('--no-input', 'Disable interactive terminal input')
|
||||
.action(async (connectionId: string, options: { force?: boolean; input?: boolean }, command) => {
|
||||
await runConnectionArgs(context, {
|
||||
command: 'remove',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
connectionId,
|
||||
force: options.force === true,
|
||||
...(options.input === false ? { inputMode: 'disabled' } : {}),
|
||||
});
|
||||
});
|
||||
|
||||
connection
|
||||
.command('map')
|
||||
.description('Refresh and validate BI-to-warehouse mappings')
|
||||
.argument('<sourceConnectionId>', 'Source BI connection id')
|
||||
.option('--json', 'Print JSON output', false)
|
||||
.action(async (sourceConnectionId: string, options: { json?: boolean }, command) => {
|
||||
await runConnectionArgs(context, {
|
||||
command: 'map',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
sourceConnectionId,
|
||||
json: options.json === true,
|
||||
});
|
||||
});
|
||||
|
||||
registerConnectionMappingCommands(connection, context);
|
||||
registerConnectionMetabaseCommands(connection, context);
|
||||
registerConnectionNotionCommands(connection, context);
|
||||
}
|
||||
|
||||
export function registerConnectionMappingCommands(connection: Command, context: KloCliCommandContext): void {
|
||||
const mapping = connection
|
||||
.command('mapping')
|
||||
.description('Manage Metabase warehouse mappings')
|
||||
.showHelpAfterError()
|
||||
.addHelpText(
|
||||
'after',
|
||||
'\nProject directory defaults to KLO_PROJECT_DIR when set, otherwise the current working directory.\n',
|
||||
);
|
||||
|
||||
mapping
|
||||
.command('list')
|
||||
.description('List Metabase database mappings')
|
||||
.argument('<connectionId>', 'Metabase connection id')
|
||||
.option('--json', 'Print JSON output where supported', false)
|
||||
.action(async (connectionId: string, options: { json?: boolean }, command) => {
|
||||
await runMappingArgs(context, {
|
||||
command: 'list',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
connectionId,
|
||||
json: options.json === true,
|
||||
});
|
||||
});
|
||||
|
||||
mapping
|
||||
.command('set')
|
||||
.description('Set a Metabase or Looker warehouse mapping')
|
||||
.argument('<connectionId>', 'Source connection id', parseSafeConnectionIdOption)
|
||||
.argument('<field>', 'Mapping field', parseMappingFieldOption)
|
||||
.argument('<assignment>', 'Mapping assignment such as 1=prod-warehouse', parseNonEmptyAssignmentOption)
|
||||
.action(
|
||||
async (
|
||||
connectionId: string,
|
||||
field: 'databaseMappings' | 'connectionMappings',
|
||||
assignment: { key: string; value: string },
|
||||
_options: unknown,
|
||||
command,
|
||||
) => {
|
||||
await runMappingArgs(context, {
|
||||
command: 'set',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
connectionId,
|
||||
field,
|
||||
key: assignment.key,
|
||||
value: assignment.value,
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
mapping
|
||||
.command('apply-bulk')
|
||||
.description('Apply mappings from JSON')
|
||||
.argument('<connectionId>', 'Metabase connection id')
|
||||
.requiredOption('--file <path>', 'JSON mapping file')
|
||||
.action(async (connectionId: string, options: { file: string }, command) => {
|
||||
await runMappingArgs(context, {
|
||||
command: 'apply-bulk',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
connectionId,
|
||||
filePath: options.file,
|
||||
});
|
||||
});
|
||||
|
||||
mapping
|
||||
.command('set-sync-enabled')
|
||||
.description('Enable or disable sync for one Metabase database')
|
||||
.argument('<connectionId>', 'Metabase connection id')
|
||||
.argument('<metabaseDatabaseId>', 'Metabase database id', parsePositiveIntegerOption)
|
||||
.requiredOption('--enabled <value>', 'true or false', parseBooleanStringOption)
|
||||
.action(
|
||||
async (connectionId: string, metabaseDatabaseId: number, options: { enabled: boolean }, command) => {
|
||||
await runMappingArgs(context, {
|
||||
command: 'set-sync-enabled',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
connectionId,
|
||||
metabaseDatabaseId,
|
||||
enabled: options.enabled,
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
const syncState = mapping.command('sync-state').description('Manage Metabase sync-state selection');
|
||||
syncState
|
||||
.command('get')
|
||||
.description('Read sync-state selection')
|
||||
.argument('<connectionId>', 'Metabase connection id')
|
||||
.option('--json', 'Print JSON output where supported', false)
|
||||
.action(async (connectionId: string, options: { json?: boolean }, command) => {
|
||||
await runMappingArgs(context, {
|
||||
command: 'sync-state-get',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
connectionId,
|
||||
json: options.json === true,
|
||||
});
|
||||
});
|
||||
|
||||
syncState
|
||||
.command('set')
|
||||
.description('Write sync-state selection')
|
||||
.argument('<connectionId>', 'Metabase connection id')
|
||||
.addOption(new Option('--mode <mode>', 'ALL, ONLY, or EXCEPT').choices(SYNC_MODE_CHOICES).makeOptionMandatory())
|
||||
.option('--collections <ids>', 'Comma-separated collection ids', parseCsvIds, [])
|
||||
.option('--items <ids>', 'Comma-separated item ids', parseCsvIds, [])
|
||||
.option('--tag-names <names>', 'Comma-separated tag names', parseCsvStrings, [])
|
||||
.action(async (connectionId: string, options, command) => {
|
||||
await runMappingArgs(context, {
|
||||
command: 'sync-state-set',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
connectionId,
|
||||
syncMode: options.mode,
|
||||
collectionIds: options.collections,
|
||||
itemIds: options.items,
|
||||
tagNames: options.tagNames,
|
||||
});
|
||||
});
|
||||
|
||||
mapping
|
||||
.command('refresh')
|
||||
.description('Refresh Metabase database mappings')
|
||||
.argument('<connectionId>', 'Metabase connection id')
|
||||
.option('--auto-accept', 'Accept refresh changes without prompting', false)
|
||||
.action(async (connectionId: string, options: { autoAccept?: boolean }, command) => {
|
||||
await runMappingArgs(context, {
|
||||
command: 'refresh',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
connectionId,
|
||||
autoAccept: options.autoAccept === true,
|
||||
});
|
||||
});
|
||||
|
||||
mapping
|
||||
.command('validate')
|
||||
.description('Validate Metabase database mappings')
|
||||
.argument('<connectionId>', 'Metabase connection id')
|
||||
.action(async (connectionId: string, _options: unknown, command) => {
|
||||
await runMappingArgs(context, {
|
||||
command: 'validate',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
connectionId,
|
||||
});
|
||||
});
|
||||
|
||||
mapping
|
||||
.command('clear')
|
||||
.description('Clear Metabase database mappings')
|
||||
.argument('<connectionId>', 'Metabase connection id')
|
||||
.argument('[metabaseDatabaseId]', 'Metabase database id', parsePositiveIntegerOption)
|
||||
.action(async (connectionId: string, metabaseDatabaseId: number | undefined, _options: unknown, command) => {
|
||||
await runMappingArgs(context, {
|
||||
command: 'clear',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
connectionId,
|
||||
...(metabaseDatabaseId ? { metabaseDatabaseId } : {}),
|
||||
});
|
||||
});
|
||||
}
|
||||
329
packages/cli/src/commands/connection-mapping.test.ts
Normal file
329
packages/cli/src/commands/connection-mapping.test.ts
Normal file
|
|
@ -0,0 +1,329 @@
|
|||
import { mkdtemp, rm } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { LocalMetabaseSourceStateReader } from '@klo/context/ingest';
|
||||
import { initKloProject, loadKloProject, serializeKloProjectConfig } from '@klo/context/project';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { runKloConnectionMapping } from './connection-mapping.js';
|
||||
|
||||
function makeIo() {
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
return {
|
||||
io: {
|
||||
stdout: {
|
||||
write: (chunk: string) => {
|
||||
stdout += chunk;
|
||||
},
|
||||
},
|
||||
stderr: {
|
||||
write: (chunk: string) => {
|
||||
stderr += chunk;
|
||||
},
|
||||
},
|
||||
},
|
||||
stdout: () => stdout,
|
||||
stderr: () => stderr,
|
||||
};
|
||||
}
|
||||
|
||||
describe('runKloConnectionMapping', () => {
|
||||
let tempDir: string;
|
||||
let projectDir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'klo-cli-metabase-mapping-'));
|
||||
projectDir = join(tempDir, 'project');
|
||||
await initKloProject({ projectDir, projectName: 'mapping' });
|
||||
const project = await loadKloProject({ projectDir });
|
||||
await project.fileStore.writeFile(
|
||||
'klo.yaml',
|
||||
serializeKloProjectConfig({
|
||||
...project.config,
|
||||
connections: {
|
||||
'prod-metabase': {
|
||||
driver: 'metabase',
|
||||
api_url: 'https://metabase.example.com',
|
||||
api_key_ref: 'env:METABASE_API_KEY', // pragma: allowlist secret
|
||||
},
|
||||
'prod-warehouse': {
|
||||
driver: 'postgres',
|
||||
url: 'env:WAREHOUSE_URL',
|
||||
readonly: true,
|
||||
},
|
||||
},
|
||||
}),
|
||||
'klo',
|
||||
'klo@example.com',
|
||||
'Seed Metabase mapping test connections',
|
||||
);
|
||||
});
|
||||
|
||||
async function replaceConnections(connections: Record<string, { driver: string; [key: string]: unknown }>) {
|
||||
const project = await loadKloProject({ projectDir });
|
||||
await project.fileStore.writeFile(
|
||||
'klo.yaml',
|
||||
serializeKloProjectConfig({
|
||||
...project.config,
|
||||
connections,
|
||||
}),
|
||||
'klo',
|
||||
'klo@example.com',
|
||||
'Replace mapping test connections',
|
||||
);
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(tempDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('sets, lists, disables, and clears local Metabase mappings', async () => {
|
||||
const io = makeIo();
|
||||
await expect(
|
||||
runKloConnectionMapping(
|
||||
{
|
||||
command: 'set',
|
||||
projectDir,
|
||||
connectionId: 'prod-metabase',
|
||||
field: 'databaseMappings',
|
||||
key: '1',
|
||||
value: 'prod-warehouse',
|
||||
},
|
||||
io.io,
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
const listIo = makeIo();
|
||||
await expect(
|
||||
runKloConnectionMapping({ command: 'list', projectDir, connectionId: 'prod-metabase', json: false }, listIo.io),
|
||||
).resolves.toBe(0);
|
||||
expect(listIo.stdout()).toContain('1 -> prod-warehouse');
|
||||
expect(listIo.stdout()).toContain('unhydrated');
|
||||
|
||||
await expect(
|
||||
runKloConnectionMapping(
|
||||
{
|
||||
command: 'set-sync-enabled',
|
||||
projectDir,
|
||||
connectionId: 'prod-metabase',
|
||||
metabaseDatabaseId: 1,
|
||||
enabled: false,
|
||||
},
|
||||
makeIo().io,
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
await expect(
|
||||
runKloConnectionMapping(
|
||||
{
|
||||
command: 'clear',
|
||||
projectDir,
|
||||
connectionId: 'prod-metabase',
|
||||
metabaseDatabaseId: 1,
|
||||
},
|
||||
makeIo().io,
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
});
|
||||
|
||||
it('lists Metabase yaml mapping bootstrap rows before any SQLite command writes', async () => {
|
||||
const projectDir = await mkdtemp(join(tmpdir(), 'klo-cli-yaml-mapping-'));
|
||||
await initKloProject({ projectDir, projectName: 'yaml-mapping' });
|
||||
const project = await loadKloProject({ projectDir });
|
||||
await project.fileStore.writeFile(
|
||||
'klo.yaml',
|
||||
serializeKloProjectConfig({
|
||||
...project.config,
|
||||
connections: {
|
||||
'prod-metabase': {
|
||||
driver: 'metabase',
|
||||
mappings: {
|
||||
databaseMappings: { '1': 'prod-warehouse' },
|
||||
syncEnabled: { '1': true },
|
||||
},
|
||||
},
|
||||
'prod-warehouse': { driver: 'postgres', url: 'postgresql://readonly@db.test/analytics' },
|
||||
},
|
||||
}),
|
||||
'klo',
|
||||
'klo@example.com',
|
||||
'Seed yaml mappings',
|
||||
);
|
||||
const io = makeIo();
|
||||
|
||||
await expect(
|
||||
runKloConnectionMapping(
|
||||
{ command: 'list', projectDir, connectionId: 'prod-metabase', json: false },
|
||||
io.io,
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(io.stdout()).toContain('1 -> prod-warehouse');
|
||||
expect(io.stdout()).toContain('source: klo.yaml');
|
||||
});
|
||||
|
||||
it('refreshes Metabase discovery metadata through the injected runtime client', async () => {
|
||||
const client = {
|
||||
getDatabases: vi.fn().mockResolvedValue([
|
||||
{
|
||||
id: 1,
|
||||
name: 'Analytics',
|
||||
engine: 'postgres',
|
||||
details: { host: 'pg.internal', dbname: 'analytics' },
|
||||
is_sample: false,
|
||||
},
|
||||
]),
|
||||
cleanup: vi.fn(),
|
||||
};
|
||||
const io = makeIo();
|
||||
|
||||
await expect(
|
||||
runKloConnectionMapping(
|
||||
{
|
||||
command: 'refresh',
|
||||
projectDir,
|
||||
connectionId: 'prod-metabase',
|
||||
autoAccept: true,
|
||||
},
|
||||
io.io,
|
||||
{
|
||||
createMetabaseClient: async () => client as never,
|
||||
},
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(io.stdout()).toContain('Discovery: 1 database');
|
||||
expect(client.cleanup).toHaveBeenCalledTimes(1);
|
||||
const store = new LocalMetabaseSourceStateReader({ dbPath: join(projectDir, '.klo', 'db.sqlite') });
|
||||
await expect(store.listDatabaseMappings('prod-metabase')).resolves.toMatchObject([
|
||||
{ metabaseDatabaseId: 1, metabaseDatabaseName: 'Analytics', source: 'refresh' },
|
||||
]);
|
||||
});
|
||||
|
||||
it('sets and lists Looker connection mappings', async () => {
|
||||
await replaceConnections({
|
||||
'prod-looker': {
|
||||
driver: 'looker',
|
||||
base_url: 'https://looker.example.test',
|
||||
client_id: 'id',
|
||||
},
|
||||
'prod-warehouse': {
|
||||
driver: 'postgres',
|
||||
url: 'postgresql://readonly@db.example.test/analytics',
|
||||
},
|
||||
});
|
||||
const io = makeIo();
|
||||
|
||||
await expect(
|
||||
runKloConnectionMapping(
|
||||
{
|
||||
command: 'set',
|
||||
projectDir,
|
||||
connectionId: 'prod-looker',
|
||||
field: 'connectionMappings',
|
||||
key: 'analytics',
|
||||
value: 'prod-warehouse',
|
||||
},
|
||||
io.io,
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
await expect(
|
||||
runKloConnectionMapping({ command: 'list', projectDir, connectionId: 'prod-looker', json: false }, io.io),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(io.stdout()).toContain('analytics -> prod-warehouse');
|
||||
});
|
||||
|
||||
it('keeps driver-specific mapping field validation in the runner', async () => {
|
||||
await replaceConnections({
|
||||
'prod-looker': { driver: 'looker', base_url: 'https://looker.example.com' },
|
||||
warehouse: { driver: 'postgres', url: 'env:WAREHOUSE_URL' },
|
||||
});
|
||||
|
||||
const io = makeIo();
|
||||
await expect(
|
||||
runKloConnectionMapping(
|
||||
{
|
||||
command: 'set',
|
||||
projectDir,
|
||||
connectionId: 'prod-looker',
|
||||
field: 'databaseMappings',
|
||||
key: '1',
|
||||
value: 'warehouse',
|
||||
},
|
||||
io.io,
|
||||
),
|
||||
).resolves.toBe(1);
|
||||
|
||||
expect(io.stderr()).toContain('Looker mapping set requires connectionMappings');
|
||||
});
|
||||
|
||||
it('refreshes Looker mapping metadata and reports drift', async () => {
|
||||
await replaceConnections({
|
||||
'prod-looker': {
|
||||
driver: 'looker',
|
||||
base_url: 'https://looker.example.test',
|
||||
client_id: 'id',
|
||||
},
|
||||
'prod-warehouse': {
|
||||
driver: 'postgres',
|
||||
url: 'postgresql://readonly@db.example.test/analytics',
|
||||
},
|
||||
});
|
||||
const io = makeIo();
|
||||
|
||||
await expect(
|
||||
runKloConnectionMapping(
|
||||
{ command: 'refresh', projectDir, connectionId: 'prod-looker', autoAccept: true },
|
||||
io.io,
|
||||
{
|
||||
createLookerClient: async () => ({
|
||||
listLookerConnections: async () => [
|
||||
{
|
||||
name: 'analytics',
|
||||
host: 'db.example.test',
|
||||
database: 'analytics',
|
||||
schema: null,
|
||||
dialect: 'postgres',
|
||||
},
|
||||
],
|
||||
cleanup: async () => {},
|
||||
}),
|
||||
},
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(io.stdout()).toContain('Discovery: 1 connection');
|
||||
expect(io.stdout()).toContain('Unmapped discovered: 1');
|
||||
});
|
||||
|
||||
it('validates Looker mappings through the canonical local warehouse descriptor', async () => {
|
||||
const projectDir = await mkdtemp(join(tmpdir(), 'klo-cli-descriptor-validation-'));
|
||||
await initKloProject({ projectDir, projectName: 'descriptor-validation' });
|
||||
const project = await loadKloProject({ projectDir });
|
||||
await project.fileStore.writeFile(
|
||||
'klo.yaml',
|
||||
serializeKloProjectConfig({
|
||||
...project.config,
|
||||
connections: {
|
||||
'prod-looker': {
|
||||
driver: 'looker',
|
||||
mappings: { connectionMappings: { analytics: 'prod-warehouse' } },
|
||||
},
|
||||
'prod-warehouse': { driver: 'postgresql', url: 'postgresql://readonly@db.test/analytics' },
|
||||
},
|
||||
}),
|
||||
'klo',
|
||||
'klo@example.com',
|
||||
'Seed descriptor validation',
|
||||
);
|
||||
const io = makeIo();
|
||||
|
||||
await expect(
|
||||
runKloConnectionMapping({ command: 'validate', projectDir, connectionId: 'prod-looker' }, io.io),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(io.stdout()).toContain('Mapping validation passed: prod-looker');
|
||||
expect(io.stderr()).toBe('');
|
||||
});
|
||||
});
|
||||
426
packages/cli/src/commands/connection-mapping.ts
Normal file
426
packages/cli/src/commands/connection-mapping.ts
Normal file
|
|
@ -0,0 +1,426 @@
|
|||
import { readFile } from 'node:fs/promises';
|
||||
import { localConnectionToWarehouseDescriptor } from '@klo/context/connections';
|
||||
import {
|
||||
DEFAULT_METABASE_CLIENT_CONFIG,
|
||||
DefaultLookerConnectionClientFactory,
|
||||
DefaultMetabaseConnectionClientFactory,
|
||||
LocalLookerRuntimeStore,
|
||||
LocalMetabaseSourceStateReader,
|
||||
computeLookerMappingDrift,
|
||||
computeMetabaseMappingDrift,
|
||||
discoverLookerConnections,
|
||||
discoverMetabaseDatabases,
|
||||
lookerCredentialsFromLocalConnection,
|
||||
metabaseRuntimeConfigFromLocalConnection,
|
||||
seedLocalMappingStateFromKloYaml,
|
||||
validateLookerMappings,
|
||||
validateMappingPhysicalMatch,
|
||||
type LookerMappingClient,
|
||||
type MetabaseRuntimeClient,
|
||||
type MetabaseSyncMode,
|
||||
} from '@klo/context/ingest';
|
||||
import { type KloLocalProject, kloLocalStateDbPath, loadKloProject } from '@klo/context/project';
|
||||
import type { KloCliIo } from '../index.js';
|
||||
import { profileMark } from '../startup-profile.js';
|
||||
|
||||
profileMark('module:commands/connection-mapping');
|
||||
|
||||
export type KloConnectionMappingArgs =
|
||||
| { command: 'list'; projectDir: string; connectionId: string; json: boolean }
|
||||
| {
|
||||
command: 'set';
|
||||
projectDir: string;
|
||||
connectionId: string;
|
||||
field: 'databaseMappings' | 'connectionMappings';
|
||||
key: string;
|
||||
value: string;
|
||||
}
|
||||
| { command: 'apply-bulk'; projectDir: string; connectionId: string; filePath: string }
|
||||
| {
|
||||
command: 'set-sync-enabled';
|
||||
projectDir: string;
|
||||
connectionId: string;
|
||||
metabaseDatabaseId: number;
|
||||
enabled: boolean;
|
||||
}
|
||||
| { command: 'sync-state-get'; projectDir: string; connectionId: string; json: boolean }
|
||||
| {
|
||||
command: 'sync-state-set';
|
||||
projectDir: string;
|
||||
connectionId: string;
|
||||
syncMode: MetabaseSyncMode;
|
||||
collectionIds: number[];
|
||||
itemIds: number[];
|
||||
tagNames: string[];
|
||||
}
|
||||
| { command: 'refresh'; projectDir: string; connectionId: string; autoAccept: boolean }
|
||||
| { command: 'validate'; projectDir: string; connectionId: string }
|
||||
| { command: 'clear'; projectDir: string; connectionId: string; metabaseDatabaseId?: number; mappingKey?: string };
|
||||
|
||||
interface KloConnectionMappingDeps {
|
||||
createMetabaseClient?: (
|
||||
project: KloLocalProject,
|
||||
connectionId: string,
|
||||
) => Promise<Pick<MetabaseRuntimeClient, 'getDatabases' | 'cleanup'>>;
|
||||
createLookerClient?: (
|
||||
project: KloLocalProject,
|
||||
connectionId: string,
|
||||
) => Promise<Pick<LookerMappingClient, 'listLookerConnections'> & { cleanup?(): Promise<void> }>;
|
||||
}
|
||||
|
||||
interface MetabaseBulkMappingPayload {
|
||||
databaseMappings?: Record<string, string | null>;
|
||||
syncEnabled?: Record<string, boolean>;
|
||||
syncMode?: MetabaseSyncMode;
|
||||
selections?: { collections?: number[]; items?: number[] };
|
||||
defaultTagNames?: string[];
|
||||
}
|
||||
|
||||
function parseId(value: string, label: string): number {
|
||||
const parsed = Number(value);
|
||||
if (!Number.isInteger(parsed) || parsed < 1) {
|
||||
throw new Error(`${label} must be a positive integer`);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
async function createDefaultMetabaseClient(
|
||||
project: KloLocalProject,
|
||||
connectionId: string,
|
||||
): Promise<Pick<MetabaseRuntimeClient, 'getDatabases' | 'cleanup'>> {
|
||||
const factory = new DefaultMetabaseConnectionClientFactory(
|
||||
(metabaseConnectionId) =>
|
||||
metabaseRuntimeConfigFromLocalConnection(metabaseConnectionId, project.config.connections[metabaseConnectionId]),
|
||||
DEFAULT_METABASE_CLIENT_CONFIG,
|
||||
);
|
||||
return factory.createClient(connectionId);
|
||||
}
|
||||
|
||||
async function createDefaultLookerClient(
|
||||
project: KloLocalProject,
|
||||
connectionId: string,
|
||||
): Promise<Pick<LookerMappingClient, 'listLookerConnections'> & { cleanup?(): Promise<void> }> {
|
||||
const factory = new DefaultLookerConnectionClientFactory({
|
||||
async resolve(lookerConnectionId) {
|
||||
return lookerCredentialsFromLocalConnection(lookerConnectionId, project.config.connections[lookerConnectionId]);
|
||||
},
|
||||
});
|
||||
return factory.createClient(connectionId) as unknown as Pick<LookerMappingClient, 'listLookerConnections'> & {
|
||||
cleanup?(): Promise<void>;
|
||||
};
|
||||
}
|
||||
|
||||
function isLookerConnection(project: KloLocalProject, connectionId: string): boolean {
|
||||
return String(project.config.connections[connectionId]?.driver ?? '').toLowerCase() === 'looker';
|
||||
}
|
||||
|
||||
function assertLookerConnection(project: KloLocalProject, connectionId: string): void {
|
||||
if (!isLookerConnection(project, connectionId)) {
|
||||
throw new Error(`Connection "${connectionId}" is not a Looker connection`);
|
||||
}
|
||||
}
|
||||
|
||||
function assertMetabaseConnection(project: KloLocalProject, connectionId: string): void {
|
||||
const connection = project.config.connections[connectionId];
|
||||
if (!connection || String(connection.driver).toLowerCase() !== 'metabase') {
|
||||
throw new Error(`Connection "${connectionId}" is not a Metabase connection`);
|
||||
}
|
||||
}
|
||||
|
||||
function assertTargetConnection(project: KloLocalProject, connectionId: string): void {
|
||||
if (!project.config.connections[connectionId]) {
|
||||
throw new Error(`Target connection "${connectionId}" does not exist`);
|
||||
}
|
||||
}
|
||||
|
||||
function targetPhysicalInfo(project: KloLocalProject, connectionId: string) {
|
||||
const descriptor = localConnectionToWarehouseDescriptor(connectionId, project.config.connections[connectionId]);
|
||||
if (!descriptor) {
|
||||
return { connection_type: 'UNKNOWN' };
|
||||
}
|
||||
return {
|
||||
connection_type: descriptor.connection_type,
|
||||
host: descriptor.host ?? null,
|
||||
database: descriptor.database ?? null,
|
||||
account: descriptor.account ?? null,
|
||||
project_id: descriptor.project_id ?? null,
|
||||
dataset_id: descriptor.dataset_id ?? null,
|
||||
...descriptor.connection_params,
|
||||
};
|
||||
}
|
||||
|
||||
function renderMapping(
|
||||
row: Awaited<ReturnType<LocalMetabaseSourceStateReader['listDatabaseMappings']>>[number],
|
||||
): string {
|
||||
const name = row.metabaseDatabaseName ?? 'unhydrated';
|
||||
const target = row.targetConnectionId ?? '[unmapped]';
|
||||
return `${row.metabaseDatabaseId} -> ${target} (${name}, sync: ${row.syncEnabled ? 'on' : 'off'}, source: ${
|
||||
row.source
|
||||
})`;
|
||||
}
|
||||
|
||||
function renderLookerMapping(row: Awaited<ReturnType<LocalLookerRuntimeStore['listConnectionMappings']>>[number]): string {
|
||||
const target = row.kloConnectionId ?? '[unmapped]';
|
||||
const metadata = [row.lookerDialect, row.lookerHost, row.lookerDatabase].filter(Boolean).join(', ');
|
||||
return `${row.lookerConnectionName} -> ${target}${metadata ? ` (${metadata}, source: ${row.source})` : ` (source: ${row.source})`}`;
|
||||
}
|
||||
|
||||
export async function runKloConnectionMapping(
|
||||
args: KloConnectionMappingArgs,
|
||||
io: KloCliIo = process,
|
||||
deps: KloConnectionMappingDeps = {},
|
||||
): Promise<number> {
|
||||
try {
|
||||
const project = await loadKloProject({ projectDir: args.projectDir });
|
||||
await seedLocalMappingStateFromKloYaml(project, args.connectionId);
|
||||
if (isLookerConnection(project, args.connectionId)) {
|
||||
assertLookerConnection(project, args.connectionId);
|
||||
const store = new LocalLookerRuntimeStore({ dbPath: kloLocalStateDbPath(project) });
|
||||
|
||||
if (args.command === 'list') {
|
||||
const rows = await store.listConnectionMappings(args.connectionId);
|
||||
io.stdout.write(args.json ? `${JSON.stringify(rows, null, 2)}\n` : `${rows.map(renderLookerMapping).join('\n')}\n`);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (args.command === 'set') {
|
||||
if (args.field !== 'connectionMappings') {
|
||||
throw new Error('Looker mapping set requires connectionMappings <lookerConnectionName>=<targetConnectionId>');
|
||||
}
|
||||
assertTargetConnection(project, args.value);
|
||||
await store.upsertConnectionMapping({
|
||||
lookerConnectionId: args.connectionId,
|
||||
lookerConnectionName: args.key,
|
||||
kloConnectionId: args.value,
|
||||
source: 'cli',
|
||||
});
|
||||
io.stdout.write(`Set connectionMappings.${args.key} = ${args.value}\n`);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (args.command === 'refresh') {
|
||||
const client = await (deps.createLookerClient ?? createDefaultLookerClient)(project, args.connectionId);
|
||||
try {
|
||||
const discovered = await discoverLookerConnections(client);
|
||||
const drift = computeLookerMappingDrift({
|
||||
storedMappings: await store.readMappings(args.connectionId),
|
||||
discovered,
|
||||
});
|
||||
if (args.autoAccept) {
|
||||
await store.refreshDiscoveredConnections({ lookerConnectionId: args.connectionId, discovered });
|
||||
}
|
||||
io.stdout.write(`Discovery: ${discovered.length} ${discovered.length === 1 ? 'connection' : 'connections'}\n`);
|
||||
io.stdout.write(`Unmapped discovered: ${drift.unmappedDiscovered.length}\n`);
|
||||
io.stdout.write(`Stale mappings: ${drift.staleMappings.length}\n`);
|
||||
return 0;
|
||||
} finally {
|
||||
await client.cleanup?.();
|
||||
}
|
||||
}
|
||||
|
||||
if (args.command === 'validate') {
|
||||
const knownKloConnectionIds = new Set(Object.keys(project.config.connections));
|
||||
const knownConnectionTypes = new Map(
|
||||
Object.entries(project.config.connections).map(([id, _config]) => [id, targetPhysicalInfo(project, id).connection_type]),
|
||||
);
|
||||
const validation = validateLookerMappings({
|
||||
mappings: await store.readMappings(args.connectionId),
|
||||
knownKloConnectionIds,
|
||||
knownConnectionTypes,
|
||||
});
|
||||
if (!validation.ok) {
|
||||
for (const error of validation.errors) {
|
||||
io.stderr.write(`${error.key}: ${error.reason}\n`);
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
io.stdout.write(`Mapping validation passed: ${args.connectionId}\n`);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (args.command === 'clear') {
|
||||
await store.clearConnectionMappings({
|
||||
lookerConnectionId: args.connectionId,
|
||||
lookerConnectionName: args.mappingKey ?? (args.metabaseDatabaseId ? String(args.metabaseDatabaseId) : undefined),
|
||||
});
|
||||
io.stdout.write(
|
||||
args.mappingKey
|
||||
? `Cleared connectionMappings.${args.mappingKey}\n`
|
||||
: `Cleared mappings for ${args.connectionId}\n`,
|
||||
);
|
||||
return 0;
|
||||
}
|
||||
|
||||
throw new Error(`Looker connection mapping does not support ${args.command}`);
|
||||
}
|
||||
|
||||
assertMetabaseConnection(project, args.connectionId);
|
||||
const store = new LocalMetabaseSourceStateReader({ dbPath: kloLocalStateDbPath(project) });
|
||||
|
||||
if (args.command === 'list') {
|
||||
const rows = await store.listDatabaseMappings(args.connectionId);
|
||||
io.stdout.write(args.json ? `${JSON.stringify(rows, null, 2)}\n` : `${rows.map(renderMapping).join('\n')}\n`);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (args.command === 'set') {
|
||||
assertTargetConnection(project, args.value);
|
||||
await store.upsertDatabaseMapping({
|
||||
connectionId: args.connectionId,
|
||||
metabaseDatabaseId: parseId(args.key, 'metabaseDatabaseId'),
|
||||
targetConnectionId: args.value,
|
||||
syncEnabled: true,
|
||||
source: 'cli',
|
||||
});
|
||||
io.stdout.write(`Set databaseMappings.${args.key} = ${args.value}\n`);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (args.command === 'apply-bulk') {
|
||||
const payload = JSON.parse(await readFile(args.filePath, 'utf8')) as MetabaseBulkMappingPayload;
|
||||
const existingState = await store.getSourceState(args.connectionId);
|
||||
const existingRows = await store.listDatabaseMappings(args.connectionId);
|
||||
const existingById = new Map(existingRows.map((row) => [row.metabaseDatabaseId, row]));
|
||||
const databaseMappings = payload.databaseMappings ?? {};
|
||||
for (const targetConnectionId of Object.values(databaseMappings)) {
|
||||
if (targetConnectionId) {
|
||||
assertTargetConnection(project, targetConnectionId);
|
||||
}
|
||||
}
|
||||
const mappingIds = new Set([
|
||||
...existingRows.map((row) => row.metabaseDatabaseId),
|
||||
...Object.keys(databaseMappings).map((id) => parseId(id, 'metabaseDatabaseId')),
|
||||
...Object.keys(payload.syncEnabled ?? {}).map((id) => parseId(id, 'metabaseDatabaseId')),
|
||||
]);
|
||||
await store.replaceSourceState({
|
||||
connectionId: args.connectionId,
|
||||
syncMode: payload.syncMode ?? existingState.syncMode,
|
||||
defaultTagNames: payload.defaultTagNames ?? existingState.defaultTagNames,
|
||||
selections:
|
||||
payload.selections === undefined
|
||||
? existingState.selections
|
||||
: [
|
||||
...(payload.selections.collections ?? []).map((id) => ({
|
||||
selectionType: 'collection' as const,
|
||||
metabaseObjectId: id,
|
||||
})),
|
||||
...(payload.selections.items ?? []).map((id) => ({
|
||||
selectionType: 'item' as const,
|
||||
metabaseObjectId: id,
|
||||
})),
|
||||
],
|
||||
mappings: [...mappingIds]
|
||||
.sort((a, b) => a - b)
|
||||
.map((id) => {
|
||||
const existing = existingById.get(id);
|
||||
return {
|
||||
metabaseDatabaseId: id,
|
||||
metabaseDatabaseName: existing?.metabaseDatabaseName ?? null,
|
||||
metabaseEngine: existing?.metabaseEngine ?? null,
|
||||
metabaseHost: existing?.metabaseHost ?? null,
|
||||
metabaseDbName: existing?.metabaseDbName ?? null,
|
||||
targetConnectionId: databaseMappings[String(id)] ?? existing?.targetConnectionId ?? null,
|
||||
syncEnabled: payload.syncEnabled?.[String(id)] ?? existing?.syncEnabled ?? false,
|
||||
source: 'cli',
|
||||
};
|
||||
}),
|
||||
});
|
||||
io.stdout.write(`Applied bulk mappings for ${args.connectionId}\n`);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (args.command === 'set-sync-enabled') {
|
||||
await store.setMappingSyncEnabled({
|
||||
connectionId: args.connectionId,
|
||||
metabaseDatabaseId: args.metabaseDatabaseId,
|
||||
syncEnabled: args.enabled,
|
||||
});
|
||||
io.stdout.write(`Set syncEnabled.${args.metabaseDatabaseId} = ${args.enabled}\n`);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (args.command === 'sync-state-get') {
|
||||
const state = await store.getSourceState(args.connectionId);
|
||||
const payload = {
|
||||
syncMode: state.syncMode,
|
||||
selections: state.selections,
|
||||
defaultTagNames: state.defaultTagNames,
|
||||
};
|
||||
io.stdout.write(args.json ? `${JSON.stringify(payload, null, 2)}\n` : `${payload.syncMode}\n`);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (args.command === 'sync-state-set') {
|
||||
await store.setSyncState({
|
||||
connectionId: args.connectionId,
|
||||
syncMode: args.syncMode,
|
||||
defaultTagNames: args.tagNames,
|
||||
selections: [
|
||||
...args.collectionIds.map((id) => ({ selectionType: 'collection' as const, metabaseObjectId: id })),
|
||||
...args.itemIds.map((id) => ({ selectionType: 'item' as const, metabaseObjectId: id })),
|
||||
],
|
||||
});
|
||||
io.stdout.write(`Set sync state for ${args.connectionId}\n`);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (args.command === 'refresh') {
|
||||
const client = await (deps.createMetabaseClient ?? createDefaultMetabaseClient)(project, args.connectionId);
|
||||
try {
|
||||
const discovered = await discoverMetabaseDatabases(client);
|
||||
const existing = Object.fromEntries(
|
||||
(await store.listDatabaseMappings(args.connectionId)).map((row) => [
|
||||
String(row.metabaseDatabaseId),
|
||||
row.targetConnectionId,
|
||||
]),
|
||||
);
|
||||
const drift = computeMetabaseMappingDrift({ currentMappings: existing, discovered });
|
||||
if (args.autoAccept) {
|
||||
await store.refreshDiscoveredDatabases({ connectionId: args.connectionId, discovered });
|
||||
}
|
||||
io.stdout.write(`Discovery: ${discovered.length} ${discovered.length === 1 ? 'database' : 'databases'}\n`);
|
||||
io.stdout.write(`Unmapped discovered: ${drift.unmappedDiscovered.length}\n`);
|
||||
io.stdout.write(`Stale mappings: ${drift.staleMappings.length}\n`);
|
||||
return 0;
|
||||
} finally {
|
||||
await client.cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
if (args.command === 'validate') {
|
||||
const rows = await store.listDatabaseMappings(args.connectionId);
|
||||
const failures = rows.flatMap((row) => {
|
||||
if (!row.targetConnectionId) {
|
||||
return [];
|
||||
}
|
||||
const reason = validateMappingPhysicalMatch(
|
||||
{ metabaseEngine: row.metabaseEngine, metabaseDbName: row.metabaseDbName, metabaseHost: row.metabaseHost },
|
||||
project.config.connections[row.targetConnectionId]
|
||||
? targetPhysicalInfo(project, row.targetConnectionId)
|
||||
: { connection_type: 'UNKNOWN' },
|
||||
);
|
||||
return reason ? [`${row.metabaseDatabaseId}: ${reason}`] : [];
|
||||
});
|
||||
if (failures.length > 0) {
|
||||
for (const failure of failures) {
|
||||
io.stderr.write(`${failure}\n`);
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
io.stdout.write(`Mapping validation passed: ${args.connectionId}\n`);
|
||||
return 0;
|
||||
}
|
||||
|
||||
const metabaseDatabaseId = args.metabaseDatabaseId ?? (args.mappingKey ? parseId(args.mappingKey, 'metabaseDatabaseId') : undefined);
|
||||
await store.clearDatabaseMappings({ connectionId: args.connectionId, metabaseDatabaseId });
|
||||
io.stdout.write(
|
||||
metabaseDatabaseId
|
||||
? `Cleared databaseMappings.${metabaseDatabaseId}\n`
|
||||
: `Cleared mappings for ${args.connectionId}\n`,
|
||||
);
|
||||
return 0;
|
||||
} catch (error) {
|
||||
io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
132
packages/cli/src/commands/connection-metabase-commands.ts
Normal file
132
packages/cli/src/commands/connection-metabase-commands.ts
Normal file
|
|
@ -0,0 +1,132 @@
|
|||
import { type Command, Option } from '@commander-js/extra-typings';
|
||||
|
||||
import {
|
||||
type KloCliCommandContext,
|
||||
parseNonEmptyAssignmentOption,
|
||||
parsePositiveIntegerOption,
|
||||
parseSafeConnectionIdOption,
|
||||
resolveCommandProjectDir,
|
||||
} from '../cli-program.js';
|
||||
import {
|
||||
type KloConnectionMetabaseSetupArgs,
|
||||
type MetabaseSetupMappingAssignment,
|
||||
type MetabaseSetupSyncMode,
|
||||
runKloConnectionMetabaseSetup,
|
||||
} from './connection-metabase-setup.js';
|
||||
|
||||
const SYNC_MODE_CHOICES = ['ALL', 'ONLY', 'EXCEPT'] as const satisfies readonly MetabaseSetupSyncMode[];
|
||||
|
||||
interface ConnectionMetabaseSetupOptions {
|
||||
id?: string;
|
||||
url?: string;
|
||||
apiKey?: string;
|
||||
mintApiKey?: boolean;
|
||||
username?: string;
|
||||
password?: string;
|
||||
map: MetabaseSetupMappingAssignment[];
|
||||
sync: number[];
|
||||
syncMode: MetabaseSetupSyncMode;
|
||||
runIngest?: boolean;
|
||||
yes?: boolean;
|
||||
input?: boolean;
|
||||
}
|
||||
|
||||
function collectPositiveIntegerOption(value: string, previous: number[] = []): number[] {
|
||||
return [...previous, parsePositiveIntegerOption(value)];
|
||||
}
|
||||
|
||||
function parseMappingAssignment(value: string): MetabaseSetupMappingAssignment {
|
||||
const assignment = parseNonEmptyAssignmentOption(value);
|
||||
return {
|
||||
metabaseDatabaseId: parsePositiveIntegerOption(assignment.key),
|
||||
targetConnectionId: parseSafeConnectionIdOption(assignment.value),
|
||||
};
|
||||
}
|
||||
|
||||
function collectMappingOption(
|
||||
value: string,
|
||||
previous: MetabaseSetupMappingAssignment[] = [],
|
||||
): MetabaseSetupMappingAssignment[] {
|
||||
return [...previous, parseMappingAssignment(value)];
|
||||
}
|
||||
|
||||
async function runMetabaseSetupArgs(
|
||||
context: KloCliCommandContext,
|
||||
args: KloConnectionMetabaseSetupArgs,
|
||||
): Promise<void> {
|
||||
const runner = context.deps.connectionMetabaseSetup ?? runKloConnectionMetabaseSetup;
|
||||
context.setExitCode(await runner(args, context.io));
|
||||
}
|
||||
|
||||
export function registerConnectionMetabaseCommands(connection: Command, context: KloCliCommandContext): void {
|
||||
const metabase = connection
|
||||
.command('metabase')
|
||||
.description('Configure Metabase connections')
|
||||
.showHelpAfterError()
|
||||
.addHelpText(
|
||||
'after',
|
||||
'\nProject directory defaults to KLO_PROJECT_DIR when set, otherwise the current working directory.\n',
|
||||
);
|
||||
|
||||
metabase.action(() => {
|
||||
metabase.outputHelp();
|
||||
context.setExitCode(0);
|
||||
});
|
||||
|
||||
metabase
|
||||
.command('setup')
|
||||
.description('Guided setup for a Metabase connection')
|
||||
.option('--id <connectionId>', 'KLO connection id to write', parseSafeConnectionIdOption)
|
||||
.option('--url <url>', 'Metabase API URL')
|
||||
.addOption(new Option('--api-key <key>', 'Metabase API key').conflicts('mintApiKey'))
|
||||
.option('--mint-api-key', 'Mint a Metabase API key with credentials', false)
|
||||
.option('--username <email>', 'Metabase admin username for API-key minting')
|
||||
.option('--password <password>', 'Metabase admin password for API-key minting')
|
||||
.addHelpText(
|
||||
'after',
|
||||
'\nGuided equivalent of:\n' +
|
||||
' klo connection mapping refresh <connectionId> --auto-accept\n' +
|
||||
' klo connection mapping set <connectionId> databaseMappings <id>=<target>\n' +
|
||||
' klo connection mapping set-sync-enabled <connectionId> <id> --enabled true\n' +
|
||||
' klo ingest <connectionId>\n',
|
||||
)
|
||||
.option(
|
||||
'--map <metabaseDatabaseId=targetConnectionId>',
|
||||
'Assign a Metabase database id to a warehouse connection; repeatable',
|
||||
collectMappingOption,
|
||||
[],
|
||||
)
|
||||
.option(
|
||||
'--sync <metabaseDatabaseId>',
|
||||
'Enable Metabase sync for a discovered database; repeatable',
|
||||
collectPositiveIntegerOption,
|
||||
[],
|
||||
)
|
||||
.addOption(
|
||||
new Option('--sync-mode <mode>', 'Metabase sync selection mode')
|
||||
.choices(SYNC_MODE_CHOICES)
|
||||
.default('ALL' satisfies MetabaseSetupSyncMode),
|
||||
)
|
||||
.option('--run-ingest', 'Run ingest after setup', false)
|
||||
.option('--yes', 'Confirm and apply setup changes without prompting', false)
|
||||
.option('--no-input', 'Disable interactive terminal input')
|
||||
.showHelpAfterError()
|
||||
.action(async (options: ConnectionMetabaseSetupOptions, command) => {
|
||||
await runMetabaseSetupArgs(context, {
|
||||
command: 'setup',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
connectionId: options.id,
|
||||
url: options.url,
|
||||
apiKey: options.apiKey,
|
||||
mintApiKey: options.mintApiKey === true,
|
||||
metabaseUsername: options.username,
|
||||
metabasePassword: options.password,
|
||||
mappings: options.map,
|
||||
syncEnabledDatabaseIds: options.sync,
|
||||
syncMode: options.syncMode ?? 'ALL',
|
||||
runIngest: options.runIngest === true,
|
||||
yes: options.yes === true,
|
||||
inputMode: options.input === false ? 'disabled' : 'auto',
|
||||
});
|
||||
});
|
||||
}
|
||||
1136
packages/cli/src/commands/connection-metabase-setup.test.ts
Normal file
1136
packages/cli/src/commands/connection-metabase-setup.test.ts
Normal file
File diff suppressed because it is too large
Load diff
782
packages/cli/src/commands/connection-metabase-setup.ts
Normal file
782
packages/cli/src/commands/connection-metabase-setup.ts
Normal file
|
|
@ -0,0 +1,782 @@
|
|||
import type { Option as ClackOption } from '@clack/prompts';
|
||||
import {
|
||||
cancel,
|
||||
confirm,
|
||||
intro,
|
||||
isCancel,
|
||||
log,
|
||||
multiselect,
|
||||
note,
|
||||
outro,
|
||||
password,
|
||||
select,
|
||||
text,
|
||||
} from '@clack/prompts';
|
||||
import { localConnectionToWarehouseDescriptor } from '@klo/context/connections';
|
||||
import {
|
||||
DEFAULT_METABASE_CLIENT_CONFIG,
|
||||
DefaultMetabaseConnectionClientFactory,
|
||||
LocalMetabaseSourceStateReader,
|
||||
MetabaseClient,
|
||||
type MetabaseDatabase,
|
||||
type MetabaseRuntimeClient,
|
||||
type MetabaseSyncMode,
|
||||
metabaseRuntimeConfigFromLocalConnection,
|
||||
validateMappingPhysicalMatch,
|
||||
} from '@klo/context/ingest';
|
||||
import {
|
||||
type KloLocalProject,
|
||||
type KloProjectConnectionConfig,
|
||||
kloLocalStateDbPath,
|
||||
loadKloProject,
|
||||
serializeKloProjectConfig,
|
||||
} from '@klo/context/project';
|
||||
|
||||
import { createClackSpinner, type KloCliSpinner } from '../clack.js';
|
||||
import type { KloCliIo } from '../cli-runtime.js';
|
||||
import { withMenuOptionsSpacing, withMultiselectNavigation } from '../prompt-navigation.js';
|
||||
import { type KloPublicIngestArgs, runKloPublicIngest } from '../public-ingest.js';
|
||||
|
||||
export type KloMetabaseSetupInputMode = 'auto' | 'disabled';
|
||||
|
||||
export type MetabaseSetupSyncMode = MetabaseSyncMode;
|
||||
|
||||
type MetabaseSetupPromptOption<Value> = ClackOption<Value>;
|
||||
|
||||
export interface MetabaseSetupLogger {
|
||||
info(message: string): void;
|
||||
step(message: string): void;
|
||||
success(message: string): void;
|
||||
warn(message: string): void;
|
||||
error(message: string): void;
|
||||
}
|
||||
|
||||
export interface MetabaseSetupPromptAdapter {
|
||||
intro(title?: string): void;
|
||||
outro(message?: string): void;
|
||||
note(message: string, title: string): void;
|
||||
log: MetabaseSetupLogger;
|
||||
spinner(): KloCliSpinner;
|
||||
select<T extends string>(options: { message: string; options: Array<MetabaseSetupPromptOption<T>> }): Promise<T>;
|
||||
multiselect<Value extends number | string>(options: {
|
||||
message: string;
|
||||
options: Array<MetabaseSetupPromptOption<Value>>;
|
||||
initialValues?: Value[];
|
||||
required?: boolean;
|
||||
maxItems?: number;
|
||||
}): Promise<Value[]>;
|
||||
text(options: { message: string; placeholder?: string }): Promise<string>;
|
||||
password(options: { message: string }): Promise<string>;
|
||||
confirm(options: { message: string; initialValue?: boolean }): Promise<boolean>;
|
||||
cancel(message: string): void;
|
||||
}
|
||||
|
||||
type KloMetabaseSetupInteractiveIo = KloCliIo & {
|
||||
stdin?: { isTTY?: boolean };
|
||||
};
|
||||
|
||||
export interface MetabaseSetupMappingAssignment {
|
||||
metabaseDatabaseId: number;
|
||||
targetConnectionId: string;
|
||||
}
|
||||
|
||||
export interface MintMetabaseApiKeyArgs {
|
||||
url: string;
|
||||
username: string;
|
||||
password: string;
|
||||
}
|
||||
|
||||
export type MintMetabaseApiKey = (args: MintMetabaseApiKeyArgs, io: KloCliIo) => Promise<string>;
|
||||
|
||||
export interface KloConnectionMetabaseSetupArgs {
|
||||
command: 'setup';
|
||||
projectDir: string;
|
||||
connectionId?: string;
|
||||
url?: string;
|
||||
apiKey?: string;
|
||||
mintApiKey: boolean;
|
||||
metabaseUsername?: string;
|
||||
metabasePassword?: string;
|
||||
mappings: MetabaseSetupMappingAssignment[];
|
||||
syncEnabledDatabaseIds: number[];
|
||||
syncMode: MetabaseSetupSyncMode;
|
||||
runIngest: boolean;
|
||||
yes: boolean;
|
||||
inputMode: KloMetabaseSetupInputMode;
|
||||
}
|
||||
|
||||
export interface KloConnectionMetabaseSetupDeps {
|
||||
createMetabaseClient?: (
|
||||
project: KloLocalProject,
|
||||
connectionId: string,
|
||||
) => Promise<Pick<MetabaseRuntimeClient, 'testConnection' | 'getDatabases' | 'cleanup'>>;
|
||||
mintMetabaseApiKey?: MintMetabaseApiKey;
|
||||
prompts?: MetabaseSetupPromptAdapter;
|
||||
runPublicIngest?: (args: Extract<KloPublicIngestArgs, { command: 'run' }>, io: KloCliIo) => Promise<number>;
|
||||
}
|
||||
|
||||
function isMetabaseConnection(connection: KloProjectConnectionConfig | undefined): boolean {
|
||||
return (
|
||||
String(connection?.driver ?? '')
|
||||
.trim()
|
||||
.toLowerCase() === 'metabase'
|
||||
);
|
||||
}
|
||||
|
||||
function stringField(value: unknown): string | undefined {
|
||||
return typeof value === 'string' && value.trim().length > 0 ? value.trim() : undefined;
|
||||
}
|
||||
|
||||
function uniqueSorted(values: number[]): number[] {
|
||||
return [...new Set(values)].sort((a, b) => a - b);
|
||||
}
|
||||
|
||||
function resolveMetabaseUrl(connection: KloProjectConnectionConfig | undefined): string | undefined {
|
||||
return stringField(connection?.api_url) ?? stringField(connection?.apiUrl) ?? stringField(connection?.url);
|
||||
}
|
||||
|
||||
function resolveLiteralMetabaseApiKey(connection: KloProjectConnectionConfig | undefined): string | undefined {
|
||||
return stringField(connection?.api_key) ?? stringField(connection?.apiKey);
|
||||
}
|
||||
|
||||
function listMetabaseConnectionIds(project: KloLocalProject): string[] {
|
||||
return Object.entries(project.config.connections)
|
||||
.filter(([_connectionId, connection]) => isMetabaseConnection(connection))
|
||||
.map(([connectionId]) => connectionId)
|
||||
.sort();
|
||||
}
|
||||
|
||||
function listWarehouseConnectionIds(project: KloLocalProject): string[] {
|
||||
return Object.entries(project.config.connections)
|
||||
.filter(([connectionId, connection]) => localConnectionToWarehouseDescriptor(connectionId, connection) != null)
|
||||
.map(([connectionId]) => connectionId)
|
||||
.sort();
|
||||
}
|
||||
|
||||
function redactSecrets(message: string, secrets: string[]): string {
|
||||
let result = message;
|
||||
for (const secret of secrets) {
|
||||
if (!secret) {
|
||||
continue;
|
||||
}
|
||||
result = result.split(secret).join('[redacted]');
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
async function createDefaultMetabaseClient(
|
||||
project: KloLocalProject,
|
||||
connectionId: string,
|
||||
): Promise<Pick<MetabaseRuntimeClient, 'testConnection' | 'getDatabases' | 'cleanup'>> {
|
||||
const factory = new DefaultMetabaseConnectionClientFactory(
|
||||
(metabaseConnectionId) =>
|
||||
metabaseRuntimeConfigFromLocalConnection(metabaseConnectionId, project.config.connections[metabaseConnectionId]),
|
||||
DEFAULT_METABASE_CLIENT_CONFIG,
|
||||
);
|
||||
return factory.createClient(connectionId);
|
||||
}
|
||||
|
||||
async function defaultMintMetabaseApiKey(args: MintMetabaseApiKeyArgs): Promise<string> {
|
||||
const loginClient = new MetabaseClient({ apiUrl: args.url, apiKey: '' }, DEFAULT_METABASE_CLIENT_CONFIG);
|
||||
const sessionId = await loginClient.createSession(args.username, args.password);
|
||||
const sessionClient = new MetabaseClient(
|
||||
{ apiUrl: args.url, apiKey: sessionId, authHeaderName: 'X-Metabase-Session' },
|
||||
DEFAULT_METABASE_CLIENT_CONFIG,
|
||||
);
|
||||
const groups = await sessionClient.getPermissionGroups();
|
||||
const adminGroup = groups.find((group) => group.name === 'Administrators');
|
||||
|
||||
if (!adminGroup) {
|
||||
throw new Error('Metabase Administrators group was not found; create an API key manually and pass --api-key');
|
||||
}
|
||||
|
||||
const mintedKey = await sessionClient.createApiKey({
|
||||
groupId: adminGroup.id,
|
||||
name: `KLO CLI ${new Date().toISOString()}`,
|
||||
});
|
||||
const trimmedKey = stringField(mintedKey);
|
||||
if (!trimmedKey) {
|
||||
throw new Error('Metabase API key minting returned an empty key');
|
||||
}
|
||||
return trimmedKey;
|
||||
}
|
||||
|
||||
function ensureNotCancelled<T>(value: T | symbol, prompts: Pick<MetabaseSetupPromptAdapter, 'cancel'>): T {
|
||||
if (isCancel(value)) {
|
||||
prompts.cancel('Setup cancelled.');
|
||||
throw new Error('Setup cancelled.');
|
||||
}
|
||||
return value as T;
|
||||
}
|
||||
|
||||
export function createClackMetabaseSetupPromptAdapter(): MetabaseSetupPromptAdapter {
|
||||
return {
|
||||
intro(title?: string): void {
|
||||
intro(title);
|
||||
},
|
||||
outro(message?: string): void {
|
||||
outro(message);
|
||||
},
|
||||
note(message: string, title: string): void {
|
||||
note(message, title);
|
||||
},
|
||||
log: {
|
||||
info(message: string): void {
|
||||
log.info(message);
|
||||
},
|
||||
step(message: string): void {
|
||||
log.step(message);
|
||||
},
|
||||
success(message: string): void {
|
||||
log.success(message);
|
||||
},
|
||||
warn(message: string): void {
|
||||
log.warn(message);
|
||||
},
|
||||
error(message: string): void {
|
||||
log.error(message);
|
||||
},
|
||||
},
|
||||
spinner(): KloCliSpinner {
|
||||
return createClackSpinner();
|
||||
},
|
||||
async select<T extends string>(options: {
|
||||
message: string;
|
||||
options: Array<MetabaseSetupPromptOption<T>>;
|
||||
}): Promise<T> {
|
||||
return ensureNotCancelled(await select(withMenuOptionsSpacing(options)), this);
|
||||
},
|
||||
async multiselect<Value extends number | string>(options: {
|
||||
message: string;
|
||||
options: Array<MetabaseSetupPromptOption<Value>>;
|
||||
initialValues?: Value[];
|
||||
required?: boolean;
|
||||
maxItems?: number;
|
||||
}): Promise<Value[]> {
|
||||
return ensureNotCancelled(await multiselect(withMenuOptionsSpacing(options)), this);
|
||||
},
|
||||
async text(options: { message: string; placeholder?: string }): Promise<string> {
|
||||
return ensureNotCancelled(await text(options), this);
|
||||
},
|
||||
async password(options: { message: string }): Promise<string> {
|
||||
return ensureNotCancelled(await password(options), this);
|
||||
},
|
||||
async confirm(options: { message: string; initialValue?: boolean }): Promise<boolean> {
|
||||
return ensureNotCancelled(await confirm(options), this);
|
||||
},
|
||||
cancel(message: string): void {
|
||||
cancel(message);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function isInteractiveMetabaseSetupIo(
|
||||
args: Pick<KloConnectionMetabaseSetupArgs, 'inputMode'>,
|
||||
io: KloMetabaseSetupInteractiveIo,
|
||||
): boolean {
|
||||
return args.inputMode !== 'disabled' && io.stdin?.isTTY === true && io.stdout.isTTY === true;
|
||||
}
|
||||
|
||||
function normalizeDiscoveredDatabases(databases: MetabaseDatabase[]): Array<{
|
||||
id: number;
|
||||
name: string;
|
||||
engine: string;
|
||||
host: string | null;
|
||||
dbName: string | null;
|
||||
}> {
|
||||
return databases
|
||||
.filter((database) => database.is_sample !== true)
|
||||
.map((database) => ({
|
||||
id: database.id,
|
||||
name: database.name,
|
||||
engine: stringField(database.engine) ?? 'unknown',
|
||||
host: stringField(database.details?.host) ?? null,
|
||||
dbName: stringField(database.details?.dbname) ?? null,
|
||||
}));
|
||||
}
|
||||
|
||||
function targetPhysicalInfo(project: KloLocalProject, connectionId: string) {
|
||||
const descriptor = localConnectionToWarehouseDescriptor(connectionId, project.config.connections[connectionId]);
|
||||
if (!descriptor) {
|
||||
return { connection_type: 'UNKNOWN' };
|
||||
}
|
||||
return {
|
||||
connection_type: descriptor.connection_type,
|
||||
host: descriptor.host ?? null,
|
||||
database: descriptor.database ?? null,
|
||||
account: descriptor.account ?? null,
|
||||
project_id: descriptor.project_id ?? null,
|
||||
dataset_id: descriptor.dataset_id ?? null,
|
||||
...descriptor.connection_params,
|
||||
};
|
||||
}
|
||||
|
||||
function noteMetabaseSetupSummary(options: {
|
||||
prompts: MetabaseSetupPromptAdapter;
|
||||
connectionId: string;
|
||||
url: string;
|
||||
mappings: MetabaseSetupMappingAssignment[];
|
||||
syncEnabledDatabaseIds: number[];
|
||||
}): void {
|
||||
const mappingLines = options.mappings
|
||||
.map((mapping) => ` ${mapping.metabaseDatabaseId} -> ${mapping.targetConnectionId}`)
|
||||
.join('\n');
|
||||
const syncLines = options.syncEnabledDatabaseIds.map((id) => ` ${id}`).join('\n');
|
||||
|
||||
options.prompts.note(
|
||||
[
|
||||
`Connection: ${options.connectionId}`,
|
||||
`URL: ${options.url}`,
|
||||
'',
|
||||
'Mappings:',
|
||||
mappingLines || ' (none)',
|
||||
'',
|
||||
'Sync enabled:',
|
||||
syncLines || ' (none)',
|
||||
].join('\n'),
|
||||
'Summary',
|
||||
);
|
||||
}
|
||||
|
||||
export async function runKloConnectionMetabaseSetup(
|
||||
args: KloConnectionMetabaseSetupArgs,
|
||||
io: KloCliIo,
|
||||
deps: KloConnectionMetabaseSetupDeps = {},
|
||||
): Promise<number> {
|
||||
let apiKeyForRedaction = args.apiKey;
|
||||
let passwordForRedaction = args.metabasePassword;
|
||||
const interactiveIo = io as KloMetabaseSetupInteractiveIo;
|
||||
const isInteractive = isInteractiveMetabaseSetupIo(args, interactiveIo);
|
||||
const prompts = deps.prompts ?? (isInteractive ? createClackMetabaseSetupPromptAdapter() : undefined);
|
||||
|
||||
try {
|
||||
if (isInteractive && prompts) {
|
||||
prompts.intro('KLO Metabase setup');
|
||||
}
|
||||
|
||||
const project = await loadKloProject({ projectDir: args.projectDir });
|
||||
const existingMetabaseConnectionIds = listMetabaseConnectionIds(project);
|
||||
let connectionId: string;
|
||||
|
||||
if (args.connectionId) {
|
||||
connectionId = args.connectionId;
|
||||
} else if (existingMetabaseConnectionIds.length === 1) {
|
||||
const onlyMetabaseConnectionId = existingMetabaseConnectionIds[0];
|
||||
if (!onlyMetabaseConnectionId) {
|
||||
throw new Error('No Metabase connection id was resolved');
|
||||
}
|
||||
connectionId = onlyMetabaseConnectionId;
|
||||
} else if (existingMetabaseConnectionIds.length > 1) {
|
||||
if (!isInteractive || !prompts) {
|
||||
throw new Error(
|
||||
`Multiple Metabase connections found (${existingMetabaseConnectionIds.join(', ')}); select one with --id`,
|
||||
);
|
||||
}
|
||||
connectionId = await prompts.select({
|
||||
message: 'Select the Metabase connection to configure',
|
||||
options: existingMetabaseConnectionIds.map((id) => ({ value: id, label: id })),
|
||||
});
|
||||
} else {
|
||||
connectionId = 'metabase';
|
||||
}
|
||||
|
||||
const existingConnection = project.config.connections[connectionId];
|
||||
const warehouseConnectionIds = listWarehouseConnectionIds(project);
|
||||
|
||||
if (warehouseConnectionIds.length === 0) {
|
||||
throw new Error('Add a warehouse connection first');
|
||||
}
|
||||
|
||||
let url = args.url ?? resolveMetabaseUrl(existingConnection);
|
||||
let apiKey = args.apiKey ?? resolveLiteralMetabaseApiKey(existingConnection);
|
||||
apiKeyForRedaction = apiKey;
|
||||
|
||||
if (!url && isInteractive && prompts) {
|
||||
url = stringField(
|
||||
await prompts.text({
|
||||
message: 'Metabase API URL',
|
||||
placeholder: 'http://localhost:3000',
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
if (args.inputMode === 'disabled' && !url) {
|
||||
throw new Error('missing Metabase URL');
|
||||
}
|
||||
|
||||
if (!args.apiKey && !args.mintApiKey && apiKey && isInteractive && prompts && !args.yes) {
|
||||
const reuse = await prompts.confirm({
|
||||
message: `Reuse the existing Metabase API key from connections.${connectionId}?`,
|
||||
initialValue: true,
|
||||
});
|
||||
if (!reuse) {
|
||||
apiKey = undefined;
|
||||
apiKeyForRedaction = undefined;
|
||||
}
|
||||
}
|
||||
|
||||
if (args.mintApiKey) {
|
||||
let username = stringField(args.metabaseUsername);
|
||||
let metabasePassword = stringField(args.metabasePassword);
|
||||
|
||||
if (isInteractive && prompts) {
|
||||
if (!username) {
|
||||
username = stringField(await prompts.text({ message: 'Metabase admin username' }));
|
||||
}
|
||||
if (!metabasePassword) {
|
||||
metabasePassword = stringField(await prompts.password({ message: 'Metabase admin password' }));
|
||||
}
|
||||
}
|
||||
|
||||
if (!username) {
|
||||
throw new Error('--mint-api-key requires --username');
|
||||
}
|
||||
if (!metabasePassword) {
|
||||
throw new Error('--mint-api-key requires --password');
|
||||
}
|
||||
if (!url) {
|
||||
throw new Error('Metabase URL is required (use --url)');
|
||||
}
|
||||
|
||||
passwordForRedaction = metabasePassword;
|
||||
apiKey = await (deps.mintMetabaseApiKey ?? defaultMintMetabaseApiKey)(
|
||||
{ url, username, password: metabasePassword },
|
||||
io,
|
||||
);
|
||||
apiKeyForRedaction = apiKey;
|
||||
}
|
||||
|
||||
if (!apiKey && isInteractive && prompts) {
|
||||
const credentialMode = await prompts.select({
|
||||
message: 'Metabase credentials',
|
||||
options: [
|
||||
{ value: 'paste', label: 'Paste API key' },
|
||||
{ value: 'mint', label: 'Mint API key' },
|
||||
],
|
||||
});
|
||||
|
||||
if (credentialMode === 'paste') {
|
||||
apiKey = stringField(await prompts.password({ message: 'Metabase API key' }));
|
||||
apiKeyForRedaction = apiKey;
|
||||
} else {
|
||||
const username = stringField(await prompts.text({ message: 'Metabase admin username' }));
|
||||
const metabasePassword = stringField(await prompts.password({ message: 'Metabase admin password' }));
|
||||
if (!username) {
|
||||
throw new Error('Metabase username is required');
|
||||
}
|
||||
if (!metabasePassword) {
|
||||
throw new Error('Metabase password is required');
|
||||
}
|
||||
if (!url) {
|
||||
throw new Error('Metabase URL is required (use --url)');
|
||||
}
|
||||
|
||||
passwordForRedaction = metabasePassword;
|
||||
apiKey = await (deps.mintMetabaseApiKey ?? defaultMintMetabaseApiKey)(
|
||||
{ url, username, password: metabasePassword },
|
||||
io,
|
||||
);
|
||||
apiKeyForRedaction = apiKey;
|
||||
}
|
||||
}
|
||||
|
||||
if (args.inputMode === 'disabled' && !apiKey) {
|
||||
throw new Error('missing Metabase API key');
|
||||
}
|
||||
|
||||
if (!url) {
|
||||
throw new Error('Metabase URL is required (use --url)');
|
||||
}
|
||||
if (!apiKey) {
|
||||
throw new Error('Metabase API key is required (use --api-key)');
|
||||
}
|
||||
|
||||
const transientConnectionConfig: KloProjectConnectionConfig = {
|
||||
...(existingConnection ?? {}),
|
||||
driver: 'metabase',
|
||||
api_url: url,
|
||||
api_key: apiKey,
|
||||
};
|
||||
const configWithTransient = {
|
||||
...project.config,
|
||||
connections: {
|
||||
...project.config.connections,
|
||||
[connectionId]: transientConnectionConfig,
|
||||
},
|
||||
};
|
||||
const discoveryProject: KloLocalProject = { ...project, config: configWithTransient };
|
||||
|
||||
for (const mapping of args.mappings) {
|
||||
if (!configWithTransient.connections[mapping.targetConnectionId]) {
|
||||
throw new Error(`Target connection "${mapping.targetConnectionId}" does not exist`);
|
||||
}
|
||||
}
|
||||
|
||||
const client = await (deps.createMetabaseClient ?? createDefaultMetabaseClient)(discoveryProject, connectionId);
|
||||
try {
|
||||
const authSpinner = isInteractive && prompts ? prompts.spinner() : undefined;
|
||||
authSpinner?.start('Testing Metabase connection');
|
||||
const testResult = await client.testConnection();
|
||||
if (!testResult.success) {
|
||||
authSpinner?.error('Metabase authentication failed');
|
||||
throw new Error(
|
||||
`Metabase authentication failed. Replace connections.${connectionId}.api_key or use --mint-api-key.`,
|
||||
);
|
||||
}
|
||||
authSpinner?.stop('Metabase reachable');
|
||||
|
||||
const discoverySpinner = isInteractive && prompts ? prompts.spinner() : undefined;
|
||||
discoverySpinner?.start('Discovering Metabase databases');
|
||||
const discovered = normalizeDiscoveredDatabases(await client.getDatabases());
|
||||
discoverySpinner?.stop(`Discovered ${discovered.length} ${discovered.length === 1 ? 'database' : 'databases'}`);
|
||||
if (isInteractive && prompts) {
|
||||
prompts.log.success(
|
||||
`Discovered ${discovered.length} ${discovered.length === 1 ? 'database' : 'databases'}`,
|
||||
);
|
||||
}
|
||||
if (discovered.length === 0) {
|
||||
throw new Error('Metabase auth worked but no usable databases were returned');
|
||||
}
|
||||
|
||||
let resolvedMappings = args.mappings;
|
||||
let resolvedSyncEnabledDatabaseIds = args.syncEnabledDatabaseIds;
|
||||
|
||||
if (resolvedSyncEnabledDatabaseIds.length === 0 && args.yes && resolvedMappings.length > 0) {
|
||||
resolvedSyncEnabledDatabaseIds = uniqueSorted(resolvedMappings.map((mapping) => mapping.metabaseDatabaseId));
|
||||
}
|
||||
|
||||
if (resolvedMappings.length === 0 && resolvedSyncEnabledDatabaseIds.length === 0) {
|
||||
const onlyDiscoveredDatabase = discovered.length === 1 ? discovered[0] : undefined;
|
||||
const compatibleWarehouses = onlyDiscoveredDatabase
|
||||
? warehouseConnectionIds.filter((warehouseConnectionId) => {
|
||||
const mismatchReason = validateMappingPhysicalMatch(
|
||||
{
|
||||
metabaseEngine: onlyDiscoveredDatabase.engine,
|
||||
metabaseDbName: onlyDiscoveredDatabase.dbName,
|
||||
metabaseHost: onlyDiscoveredDatabase.host,
|
||||
},
|
||||
targetPhysicalInfo(project, warehouseConnectionId),
|
||||
);
|
||||
return !mismatchReason;
|
||||
})
|
||||
: [];
|
||||
const onlyWarehouseConnectionId = compatibleWarehouses[0];
|
||||
|
||||
if (onlyDiscoveredDatabase && compatibleWarehouses.length === 1 && onlyWarehouseConnectionId) {
|
||||
if (args.yes) {
|
||||
resolvedMappings = [
|
||||
{ metabaseDatabaseId: onlyDiscoveredDatabase.id, targetConnectionId: onlyWarehouseConnectionId },
|
||||
];
|
||||
resolvedSyncEnabledDatabaseIds = [onlyDiscoveredDatabase.id];
|
||||
} else if (isInteractive && prompts) {
|
||||
const proposedMappings = [
|
||||
{ metabaseDatabaseId: onlyDiscoveredDatabase.id, targetConnectionId: onlyWarehouseConnectionId },
|
||||
];
|
||||
const proposedSyncEnabledDatabaseIds = [onlyDiscoveredDatabase.id];
|
||||
noteMetabaseSetupSummary({
|
||||
prompts,
|
||||
connectionId,
|
||||
url,
|
||||
mappings: proposedMappings,
|
||||
syncEnabledDatabaseIds: proposedSyncEnabledDatabaseIds,
|
||||
});
|
||||
const confirmed = await prompts.confirm({
|
||||
message: `Map Metabase database "${onlyDiscoveredDatabase.name}" (${onlyDiscoveredDatabase.id}) to "${onlyWarehouseConnectionId}" and enable sync?`,
|
||||
initialValue: true,
|
||||
});
|
||||
if (!confirmed) {
|
||||
prompts.cancel('Setup cancelled.');
|
||||
throw new Error('Setup cancelled.');
|
||||
}
|
||||
resolvedMappings = proposedMappings;
|
||||
resolvedSyncEnabledDatabaseIds = proposedSyncEnabledDatabaseIds;
|
||||
} else {
|
||||
throw new Error('Metabase mapping/sync is required in --no-input mode; pass --map and --sync');
|
||||
}
|
||||
} else if (isInteractive && prompts) {
|
||||
const selectedDatabaseIds = await prompts.multiselect<number>({
|
||||
message: withMultiselectNavigation('Select Metabase databases to configure'),
|
||||
options: discovered.map((database) => ({
|
||||
value: database.id,
|
||||
label: `${database.id}: ${database.name}`,
|
||||
hint: [database.engine, database.host, database.dbName].filter(Boolean).join(' • '),
|
||||
})),
|
||||
required: true,
|
||||
});
|
||||
|
||||
resolvedMappings = [];
|
||||
for (const databaseId of selectedDatabaseIds) {
|
||||
const database = discovered.find((candidate) => candidate.id === databaseId);
|
||||
if (!database) {
|
||||
throw new Error(`Selected database id ${databaseId} was not discovered`);
|
||||
}
|
||||
|
||||
const existingMapping = args.mappings.find((mapping) => mapping.metabaseDatabaseId === databaseId);
|
||||
if (existingMapping) {
|
||||
resolvedMappings.push(existingMapping);
|
||||
continue;
|
||||
}
|
||||
|
||||
const targetConnectionId = await prompts.select({
|
||||
message: `Map Metabase database ${database.id} ("${database.name}") to which KLO connection?`,
|
||||
options: warehouseConnectionIds.map((warehouseId) => ({ value: warehouseId, label: warehouseId })),
|
||||
});
|
||||
resolvedMappings.push({ metabaseDatabaseId: databaseId, targetConnectionId });
|
||||
}
|
||||
|
||||
const syncIds = await prompts.multiselect<number>({
|
||||
message: withMultiselectNavigation('Enable sync for which databases?'),
|
||||
options: selectedDatabaseIds.map((id) => ({ value: id, label: String(id) })),
|
||||
initialValues: selectedDatabaseIds,
|
||||
required: true,
|
||||
});
|
||||
resolvedSyncEnabledDatabaseIds = uniqueSorted(syncIds);
|
||||
|
||||
if (!args.yes) {
|
||||
noteMetabaseSetupSummary({
|
||||
prompts,
|
||||
connectionId,
|
||||
url,
|
||||
mappings: resolvedMappings,
|
||||
syncEnabledDatabaseIds: resolvedSyncEnabledDatabaseIds,
|
||||
});
|
||||
const confirmed = await prompts.confirm({
|
||||
message: 'Write changes to klo.yaml and enable sync?',
|
||||
initialValue: true,
|
||||
});
|
||||
if (!confirmed) {
|
||||
prompts.cancel('Setup cancelled.');
|
||||
throw new Error('Setup cancelled.');
|
||||
}
|
||||
}
|
||||
} else if (args.inputMode === 'disabled') {
|
||||
throw new Error('Metabase mapping/sync is required in --no-input mode; pass --map and --sync');
|
||||
}
|
||||
}
|
||||
|
||||
if (
|
||||
args.inputMode === 'disabled' &&
|
||||
resolvedMappings.length > 0 &&
|
||||
resolvedSyncEnabledDatabaseIds.length === 0
|
||||
) {
|
||||
throw new Error('Metabase sync selection is required in --no-input mode; pass --sync <metabaseDatabaseId>');
|
||||
}
|
||||
|
||||
const discoveredIds = new Set(discovered.map((database) => database.id));
|
||||
for (const mapping of resolvedMappings) {
|
||||
if (!discoveredIds.has(mapping.metabaseDatabaseId)) {
|
||||
throw new Error(`Mapped database id ${mapping.metabaseDatabaseId} was not discovered`);
|
||||
}
|
||||
}
|
||||
for (const syncId of resolvedSyncEnabledDatabaseIds) {
|
||||
if (!discoveredIds.has(syncId)) {
|
||||
throw new Error(`Sync database id ${syncId} was not discovered`);
|
||||
}
|
||||
}
|
||||
|
||||
await project.fileStore.writeFile(
|
||||
'klo.yaml',
|
||||
serializeKloProjectConfig(configWithTransient),
|
||||
'klo',
|
||||
'klo@example.com',
|
||||
`Setup Metabase connection ${connectionId}`,
|
||||
);
|
||||
|
||||
const updatedProject = await loadKloProject({ projectDir: args.projectDir });
|
||||
const store = new LocalMetabaseSourceStateReader({ dbPath: kloLocalStateDbPath(updatedProject) });
|
||||
|
||||
await store.refreshDiscoveredDatabases({ connectionId, discovered });
|
||||
|
||||
for (const mapping of resolvedMappings) {
|
||||
await store.upsertDatabaseMapping({
|
||||
connectionId,
|
||||
metabaseDatabaseId: mapping.metabaseDatabaseId,
|
||||
targetConnectionId: mapping.targetConnectionId,
|
||||
syncEnabled: false,
|
||||
source: 'cli',
|
||||
});
|
||||
}
|
||||
|
||||
for (const metabaseDatabaseId of resolvedSyncEnabledDatabaseIds) {
|
||||
await store.setMappingSyncEnabled({
|
||||
connectionId,
|
||||
metabaseDatabaseId,
|
||||
syncEnabled: true,
|
||||
});
|
||||
}
|
||||
|
||||
const existingSyncState = await store.getSourceState(connectionId);
|
||||
await store.setSyncState({
|
||||
connectionId,
|
||||
syncMode: args.syncMode,
|
||||
defaultTagNames: existingSyncState.defaultTagNames,
|
||||
selections: existingSyncState.selections,
|
||||
});
|
||||
|
||||
const unhydrated = await store.getUnhydratedSyncEnabledMappingIds(connectionId);
|
||||
if (unhydrated.length > 0) {
|
||||
io.stderr.write(
|
||||
`Sync-enabled mappings are missing discovery metadata; run klo connection mapping refresh ${connectionId} --auto-accept\n`,
|
||||
);
|
||||
return 1;
|
||||
}
|
||||
|
||||
const rows = await store.listDatabaseMappings(connectionId);
|
||||
const physicalFailures = rows.flatMap((row) => {
|
||||
if (!row.targetConnectionId) {
|
||||
return [];
|
||||
}
|
||||
const reason = validateMappingPhysicalMatch(
|
||||
{ metabaseEngine: row.metabaseEngine, metabaseDbName: row.metabaseDbName, metabaseHost: row.metabaseHost },
|
||||
updatedProject.config.connections[row.targetConnectionId]
|
||||
? targetPhysicalInfo(updatedProject, row.targetConnectionId)
|
||||
: { connection_type: 'UNKNOWN' },
|
||||
);
|
||||
return reason ? [`${row.metabaseDatabaseId}: ${reason}`] : [];
|
||||
});
|
||||
if (physicalFailures.length > 0) {
|
||||
for (const failure of physicalFailures) {
|
||||
io.stderr.write(`${failure}\n`);
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
io.stdout.write(`Connection: ${connectionId}\n`);
|
||||
io.stdout.write(`Discovered ${discovered.length} ${discovered.length === 1 ? 'database' : 'databases'}\n`);
|
||||
io.stdout.write(`Next: klo ingest ${connectionId} --project-dir ${args.projectDir}\n`);
|
||||
|
||||
if (args.runIngest) {
|
||||
const ingestRunner = deps.runPublicIngest ?? runKloPublicIngest;
|
||||
const exitCode = await ingestRunner(
|
||||
{
|
||||
command: 'run',
|
||||
projectDir: args.projectDir,
|
||||
targetConnectionId: connectionId,
|
||||
all: false,
|
||||
json: false,
|
||||
inputMode: 'disabled',
|
||||
},
|
||||
io,
|
||||
);
|
||||
if (exitCode !== 0) {
|
||||
io.stderr.write(`Ingest failed; re-run: klo ingest ${connectionId} --project-dir ${args.projectDir}\n`);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (isInteractive && prompts) {
|
||||
prompts.outro('Metabase setup complete');
|
||||
}
|
||||
|
||||
return 0;
|
||||
} finally {
|
||||
await client.cleanup();
|
||||
}
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
io.stderr.write(
|
||||
`${redactSecrets(message, [apiKeyForRedaction ?? '', passwordForRedaction ?? '', args.apiKey ?? ''])}\n`,
|
||||
);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
92
packages/cli/src/commands/connection-notion-commands.ts
Normal file
92
packages/cli/src/commands/connection-notion-commands.ts
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
import { type Command, InvalidArgumentError } from '@commander-js/extra-typings';
|
||||
import { collectOption, type KloCliCommandContext, resolveCommandProjectDir } from '../cli-program.js';
|
||||
import type { KloConnectionNotionArgs } from './connection-notion.js';
|
||||
|
||||
interface NotionPickOptions {
|
||||
input?: boolean;
|
||||
rootPageId: string[];
|
||||
}
|
||||
|
||||
function parseSafeConnectionId(value: string): string {
|
||||
if (!/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(value)) {
|
||||
throw new InvalidArgumentError(`Unsafe connection id: ${value}`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function uniqueInOrder(values: string[]): string[] {
|
||||
const seen = new Set<string>();
|
||||
const result: string[] = [];
|
||||
for (const value of values) {
|
||||
if (!seen.has(value)) {
|
||||
seen.add(value);
|
||||
result.push(value);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function normalizeNotionPageId(value: string): string {
|
||||
const trimmed = value.trim();
|
||||
const compact = trimmed.includes('-') ? trimmed.replace(/-/g, '') : trimmed;
|
||||
if (!/^[0-9a-fA-F]{32}$/.test(compact)) {
|
||||
throw new Error(`Invalid Notion page UUID: ${value}`);
|
||||
}
|
||||
const lower = compact.toLowerCase();
|
||||
return `${lower.slice(0, 8)}-${lower.slice(8, 12)}-${lower.slice(12, 16)}-${lower.slice(16, 20)}-${lower.slice(20)}`;
|
||||
}
|
||||
|
||||
function buildPickArgs(connectionId: string, projectDir: string, options: NotionPickOptions): KloConnectionNotionArgs {
|
||||
if (options.input !== false) {
|
||||
return {
|
||||
command: 'pick',
|
||||
projectDir,
|
||||
connectionId,
|
||||
mode: 'interactive',
|
||||
};
|
||||
}
|
||||
|
||||
const rootPageIds = uniqueInOrder(options.rootPageId.map(normalizeNotionPageId));
|
||||
if (rootPageIds.length === 0) {
|
||||
throw new Error('connection notion pick --no-input requires at least one --root-page-id');
|
||||
}
|
||||
return {
|
||||
command: 'pick',
|
||||
projectDir,
|
||||
connectionId,
|
||||
mode: 'non-interactive',
|
||||
rootPageIds,
|
||||
};
|
||||
}
|
||||
|
||||
async function runConnectionNotionArgs(context: KloCliCommandContext, args: KloConnectionNotionArgs): Promise<void> {
|
||||
const runner = context.deps.connectionNotion ?? (await import('./connection-notion.js')).runKloConnectionNotion;
|
||||
context.setExitCode(await runner(args, context.io));
|
||||
}
|
||||
|
||||
export function registerConnectionNotionCommands(connect: Command, context: KloCliCommandContext): void {
|
||||
const notion = connect
|
||||
.command('notion')
|
||||
.description('Configure Notion source selection')
|
||||
.showHelpAfterError()
|
||||
.addHelpText(
|
||||
'after',
|
||||
'\nProject directory defaults to KLO_PROJECT_DIR when set, otherwise the current working directory.\n',
|
||||
);
|
||||
|
||||
notion.action(() => {
|
||||
notion.outputHelp();
|
||||
context.setExitCode(0);
|
||||
});
|
||||
|
||||
notion
|
||||
.command('pick')
|
||||
.description('Pick Notion root pages for a configured Notion connection')
|
||||
.argument('<connectionId>', 'Notion connection id', parseSafeConnectionId)
|
||||
.option('--no-input', 'Disable interactive terminal input')
|
||||
.option('--root-page-id <id>', 'Root page UUID to crawl; repeatable with --no-input', collectOption, [])
|
||||
.showHelpAfterError()
|
||||
.action(async (connectionId: string, options: NotionPickOptions, command) => {
|
||||
await runConnectionNotionArgs(context, buildPickArgs(connectionId, resolveCommandProjectDir(command), options));
|
||||
});
|
||||
}
|
||||
283
packages/cli/src/commands/connection-notion-tree.test.ts
Normal file
283
packages/cli/src/commands/connection-notion-tree.test.ts
Normal file
|
|
@ -0,0 +1,283 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
buildInitialState,
|
||||
buildPickerTree,
|
||||
canToggle,
|
||||
clearExpiredTransientHint,
|
||||
filterTree,
|
||||
flattenSelection,
|
||||
moveCursor,
|
||||
reducer,
|
||||
selectAllVisible,
|
||||
selectNone,
|
||||
toggleChecked,
|
||||
TRANSIENT_HINT_DURATION_MS,
|
||||
visibleNodeIds,
|
||||
type NotionPickerPageInput,
|
||||
} from './connection-notion-tree.js';
|
||||
|
||||
const IDS = {
|
||||
engineering: '11111111-1111-1111-1111-111111111111',
|
||||
architecture: '22222222-2222-2222-2222-222222222222',
|
||||
onboarding: '33333333-3333-3333-3333-333333333333',
|
||||
marketing: '44444444-4444-4444-4444-444444444444',
|
||||
journal: '55555555-5555-5555-5555-555555555555',
|
||||
orphan: '66666666-6666-6666-6666-666666666666',
|
||||
duplicate: '77777777-7777-7777-7777-777777777777',
|
||||
cycleA: '88888888-8888-8888-8888-888888888888',
|
||||
cycleB: '99999999-9999-9999-9999-999999999999',
|
||||
};
|
||||
|
||||
function pages(): NotionPickerPageInput[] {
|
||||
return [
|
||||
{ id: IDS.marketing, title: 'Marketing', archived: false, parentId: null },
|
||||
{ id: IDS.onboarding, title: 'Onboarding', archived: false, parentId: IDS.engineering },
|
||||
{ id: IDS.engineering, title: 'Engineering Docs', archived: false, parentId: null },
|
||||
{ id: IDS.architecture, title: 'Architecture', archived: false, parentId: IDS.engineering },
|
||||
{ id: IDS.journal, title: 'Daily journal', archived: true, parentId: IDS.marketing },
|
||||
{ id: IDS.orphan, title: '', archived: false, parentId: 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa' },
|
||||
{ id: IDS.duplicate, title: 'Original duplicate', archived: false, parentId: null },
|
||||
{ id: IDS.duplicate, title: 'Ignored duplicate', archived: true, parentId: IDS.marketing },
|
||||
{ id: IDS.cycleA, title: 'Cycle A', archived: false, parentId: IDS.cycleB },
|
||||
{ id: IDS.cycleB, title: 'Cycle B', archived: false, parentId: IDS.cycleA },
|
||||
];
|
||||
}
|
||||
|
||||
describe('buildPickerTree', () => {
|
||||
it('deduplicates pages, sorts siblings, preserves archived flags, roots orphans, and breaks cycles', () => {
|
||||
const tree = buildPickerTree(pages());
|
||||
const byId = new Map(tree.map((node) => [node.id, node]));
|
||||
|
||||
expect(tree.map((node) => node.title)).toEqual([
|
||||
'Cycle A',
|
||||
'Cycle B',
|
||||
'Engineering Docs',
|
||||
'Architecture',
|
||||
'Onboarding',
|
||||
'Marketing',
|
||||
'Daily journal',
|
||||
'Original duplicate',
|
||||
'Untitled',
|
||||
]);
|
||||
expect(byId.get(IDS.engineering)?.childIds).toEqual([IDS.architecture, IDS.onboarding]);
|
||||
expect(byId.get(IDS.architecture)).toMatchObject({
|
||||
depth: 1,
|
||||
parentId: IDS.engineering,
|
||||
path: 'Engineering Docs / Architecture',
|
||||
});
|
||||
expect(byId.get(IDS.journal)).toMatchObject({
|
||||
archived: true,
|
||||
depth: 1,
|
||||
path: 'Marketing / Daily journal',
|
||||
});
|
||||
expect(byId.get(IDS.orphan)).toMatchObject({
|
||||
title: 'Untitled',
|
||||
parentId: null,
|
||||
depth: 0,
|
||||
path: 'Untitled',
|
||||
});
|
||||
expect(byId.get(IDS.duplicate)).toMatchObject({
|
||||
title: 'Original duplicate',
|
||||
archived: false,
|
||||
parentId: null,
|
||||
});
|
||||
expect(byId.get(IDS.cycleA)?.parentId).toBeNull();
|
||||
expect(byId.get(IDS.cycleB)?.parentId).toBe(IDS.cycleA);
|
||||
});
|
||||
});
|
||||
|
||||
describe('selection invariants', () => {
|
||||
it('checking a parent locks descendants and keeps checked ids minimal', () => {
|
||||
const state = buildInitialState({
|
||||
tree: buildPickerTree(pages()),
|
||||
existingRootPageIds: [],
|
||||
currentCrawlMode: 'selected_roots',
|
||||
});
|
||||
|
||||
const checkedParent = toggleChecked(state, IDS.engineering, 1000);
|
||||
expect([...checkedParent.checked]).toEqual([IDS.engineering]);
|
||||
expect(canToggle(IDS.architecture, checkedParent)).toEqual({
|
||||
ok: false,
|
||||
reason: "Locked by 'Engineering Docs' - uncheck parent first",
|
||||
});
|
||||
|
||||
const lockedChildAttempt = toggleChecked(checkedParent, IDS.architecture, 2000);
|
||||
expect([...lockedChildAttempt.checked]).toEqual([IDS.engineering]);
|
||||
expect(lockedChildAttempt.transientHint).toEqual({
|
||||
text: "Locked by 'Engineering Docs' - uncheck parent first",
|
||||
expiresAt: 4500,
|
||||
});
|
||||
|
||||
const uncheckedParent = toggleChecked(lockedChildAttempt, IDS.engineering, 3000);
|
||||
expect([...uncheckedParent.checked]).toEqual([]);
|
||||
expect(canToggle(IDS.architecture, uncheckedParent)).toEqual({ ok: true });
|
||||
});
|
||||
|
||||
it('normalizes stored roots, reports stale roots, expands checked ancestors, and flattens descendants', () => {
|
||||
const state = buildInitialState({
|
||||
tree: buildPickerTree(pages()),
|
||||
existingRootPageIds: [
|
||||
IDS.engineering.replaceAll('-', ''),
|
||||
IDS.architecture,
|
||||
'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa',
|
||||
],
|
||||
currentCrawlMode: 'selected_roots',
|
||||
});
|
||||
|
||||
expect([...state.checked]).toEqual([IDS.engineering]);
|
||||
expect([...state.expanded]).toEqual([]);
|
||||
expect(state.cursorId).toBe(IDS.cycleA);
|
||||
expect(state.preLoadWarnings).toEqual(['1 stored root_page_ids no longer visible']);
|
||||
expect(flattenSelection(new Set([IDS.engineering, IDS.architecture]), state.byId)).toEqual([IDS.engineering]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('search and cursor movement', () => {
|
||||
it('filters by title and path while deriving auto-expanded ancestors', () => {
|
||||
const state = buildInitialState({
|
||||
tree: buildPickerTree(pages()),
|
||||
existingRootPageIds: [],
|
||||
currentCrawlMode: 'selected_roots',
|
||||
});
|
||||
const searching = {
|
||||
...state,
|
||||
search: { editing: false, query: 'architecture' },
|
||||
};
|
||||
|
||||
expect(filterTree(searching)).toEqual({
|
||||
visibleIds: new Set([IDS.engineering, IDS.architecture]),
|
||||
autoExpand: new Set([IDS.engineering]),
|
||||
});
|
||||
expect(visibleNodeIds(searching)).toEqual([IDS.engineering, IDS.architecture]);
|
||||
});
|
||||
|
||||
it('moves the cursor through visible nodes and implements left/right tree semantics', () => {
|
||||
const state = buildInitialState({
|
||||
tree: buildPickerTree(pages()),
|
||||
existingRootPageIds: [],
|
||||
currentCrawlMode: 'selected_roots',
|
||||
});
|
||||
|
||||
const atEngineering = {
|
||||
...state,
|
||||
cursorId: IDS.engineering,
|
||||
expanded: new Set([IDS.engineering]),
|
||||
};
|
||||
expect(moveCursor(atEngineering, 'down').cursorId).toBe(IDS.architecture);
|
||||
expect(moveCursor({ ...atEngineering, cursorId: IDS.architecture }, 'up').cursorId).toBe(IDS.engineering);
|
||||
expect(moveCursor(atEngineering, 'right').cursorId).toBe(IDS.architecture);
|
||||
expect(moveCursor({ ...atEngineering, cursorId: IDS.architecture }, 'left').cursorId).toBe(IDS.engineering);
|
||||
expect([...moveCursor(atEngineering, 'left').expanded]).toEqual([]);
|
||||
expect([...moveCursor({ ...state, cursorId: IDS.marketing }, 'right').expanded]).toContain(IDS.marketing);
|
||||
});
|
||||
});
|
||||
|
||||
describe('bulk actions and reducer effects', () => {
|
||||
it('selects only matching visible roots under search and clears selection', () => {
|
||||
const state = buildInitialState({
|
||||
tree: buildPickerTree(pages()),
|
||||
existingRootPageIds: [IDS.marketing],
|
||||
currentCrawlMode: 'selected_roots',
|
||||
});
|
||||
const searching = {
|
||||
...state,
|
||||
search: { editing: false, query: 'architecture' },
|
||||
};
|
||||
|
||||
const selected = selectAllVisible(searching);
|
||||
expect(flattenSelection(selected.checked, selected.byId)).toEqual([IDS.architecture, IDS.marketing]);
|
||||
expect([...selectNone(selected).checked]).toEqual([]);
|
||||
});
|
||||
|
||||
it('returns save immediately for selected_roots and requires confirmation for all_accessible', () => {
|
||||
const selectedRoots = toggleChecked(
|
||||
buildInitialState({
|
||||
tree: buildPickerTree(pages()),
|
||||
existingRootPageIds: [],
|
||||
currentCrawlMode: 'selected_roots',
|
||||
}),
|
||||
IDS.marketing,
|
||||
1000,
|
||||
);
|
||||
expect(reducer(selectedRoots, 'save-request')).toEqual({
|
||||
next: selectedRoots,
|
||||
effect: 'save',
|
||||
});
|
||||
|
||||
const allAccessible = {
|
||||
...selectedRoots,
|
||||
currentCrawlMode: 'all_accessible' as const,
|
||||
};
|
||||
const confirm = reducer(allAccessible, 'save-request');
|
||||
expect(confirm).toEqual({
|
||||
next: { ...allAccessible, pendingConfirm: 'mode-switch' },
|
||||
effect: null,
|
||||
});
|
||||
expect(reducer(confirm.next, 'save-cancel')).toEqual({
|
||||
next: { ...allAccessible, pendingConfirm: null },
|
||||
effect: null,
|
||||
});
|
||||
expect(reducer(confirm.next, 'save-confirm')).toEqual({
|
||||
next: { ...allAccessible, pendingConfirm: null },
|
||||
effect: 'save',
|
||||
});
|
||||
});
|
||||
|
||||
it('blocks empty saves, updates search state, and quits without saving', () => {
|
||||
const state = buildInitialState({
|
||||
tree: buildPickerTree(pages()),
|
||||
existingRootPageIds: [],
|
||||
currentCrawlMode: 'selected_roots',
|
||||
});
|
||||
|
||||
const blockedSave = reducer(state, 'save-request', 9000);
|
||||
expect(blockedSave).toEqual({
|
||||
next: {
|
||||
...state,
|
||||
transientHint: {
|
||||
text: 'Select at least one page or press q to quit',
|
||||
expiresAt: 9000 + TRANSIENT_HINT_DURATION_MS,
|
||||
},
|
||||
},
|
||||
effect: null,
|
||||
});
|
||||
expect(
|
||||
reducer(
|
||||
reducer(reducer(state, 'search-start').next, { type: 'search-input', value: 'a' }).next,
|
||||
'search-submit',
|
||||
).next.search,
|
||||
).toEqual({ editing: false, query: 'a' });
|
||||
expect(reducer(state, 'quit')).toEqual({
|
||||
next: state,
|
||||
effect: 'quit-without-save',
|
||||
});
|
||||
});
|
||||
|
||||
it('clears transient hints only when their expiry time has passed', () => {
|
||||
const state = buildInitialState({
|
||||
tree: buildPickerTree(pages()),
|
||||
existingRootPageIds: [],
|
||||
currentCrawlMode: 'selected_roots',
|
||||
});
|
||||
const withHint = {
|
||||
...state,
|
||||
transientHint: {
|
||||
text: 'Select at least one page or press q to quit',
|
||||
expiresAt: 11500,
|
||||
},
|
||||
};
|
||||
|
||||
expect(clearExpiredTransientHint(withHint, 11499)).toBe(withHint);
|
||||
expect(clearExpiredTransientHint(withHint, 11500)).toEqual({
|
||||
...withHint,
|
||||
transientHint: null,
|
||||
});
|
||||
expect(reducer(withHint, 'clear-transient-hint', 11501)).toEqual({
|
||||
next: {
|
||||
...withHint,
|
||||
transientHint: null,
|
||||
},
|
||||
effect: null,
|
||||
});
|
||||
});
|
||||
});
|
||||
529
packages/cli/src/commands/connection-notion-tree.ts
Normal file
529
packages/cli/src/commands/connection-notion-tree.ts
Normal file
|
|
@ -0,0 +1,529 @@
|
|||
export interface NotionPickerPageInput {
|
||||
id: string;
|
||||
title?: string | null;
|
||||
archived?: boolean;
|
||||
parentId?: string | null;
|
||||
}
|
||||
|
||||
interface NotionPickerNode {
|
||||
id: string;
|
||||
title: string;
|
||||
archived: boolean;
|
||||
parentId: string | null;
|
||||
depth: number;
|
||||
childIds: string[];
|
||||
path: string;
|
||||
}
|
||||
|
||||
export interface PickerState {
|
||||
tree: NotionPickerNode[];
|
||||
byId: Map<string, NotionPickerNode>;
|
||||
expanded: Set<string>;
|
||||
checked: Set<string>;
|
||||
cursorId: string;
|
||||
search: { editing: boolean; query: string };
|
||||
pendingConfirm: 'mode-switch' | null;
|
||||
preLoadWarnings: string[];
|
||||
transientHint: { text: string; expiresAt: number } | null;
|
||||
currentCrawlMode: 'all_accessible' | 'selected_roots';
|
||||
}
|
||||
|
||||
export type PickerCommand =
|
||||
| 'cursor-up'
|
||||
| 'cursor-down'
|
||||
| 'cursor-left'
|
||||
| 'cursor-right'
|
||||
| 'expand'
|
||||
| 'collapse'
|
||||
| 'expand-all'
|
||||
| 'collapse-all'
|
||||
| 'toggle-check'
|
||||
| 'select-all-visible'
|
||||
| 'select-none'
|
||||
| 'clear-transient-hint'
|
||||
| 'search-start'
|
||||
| 'search-cancel'
|
||||
| 'search-submit'
|
||||
| 'search-backspace'
|
||||
| { type: 'search-input'; value: string }
|
||||
| 'save-request'
|
||||
| 'save-confirm'
|
||||
| 'save-cancel'
|
||||
| 'quit';
|
||||
|
||||
type PickerEffect = null | 'save' | 'quit-without-save';
|
||||
|
||||
interface MutableNode {
|
||||
id: string;
|
||||
title: string;
|
||||
archived: boolean;
|
||||
parentId: string | null;
|
||||
childIds: string[];
|
||||
}
|
||||
|
||||
export const TRANSIENT_HINT_DURATION_MS = 2500;
|
||||
|
||||
const collator = new Intl.Collator('en', { sensitivity: 'base', numeric: true });
|
||||
|
||||
function normalizePageId(value: string): string {
|
||||
const trimmed = value.trim();
|
||||
const compact = trimmed.replace(/-/g, '');
|
||||
if (/^[0-9a-fA-F]{32}$/.test(compact)) {
|
||||
const lower = compact.toLowerCase();
|
||||
return `${lower.slice(0, 8)}-${lower.slice(8, 12)}-${lower.slice(12, 16)}-${lower.slice(
|
||||
16,
|
||||
20,
|
||||
)}-${lower.slice(20)}`;
|
||||
}
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
function titleValue(value: string | null | undefined): string {
|
||||
const trimmed = value?.trim() ?? '';
|
||||
return trimmed.length > 0 ? trimmed : 'Untitled';
|
||||
}
|
||||
|
||||
function sortedNodeIds(ids: string[], nodes: Map<string, MutableNode | NotionPickerNode>): string[] {
|
||||
return [...ids].sort((leftId, rightId) => {
|
||||
const left = nodes.get(leftId);
|
||||
const right = nodes.get(rightId);
|
||||
const byTitle = collator.compare(left?.title ?? '', right?.title ?? '');
|
||||
return byTitle === 0 ? leftId.localeCompare(rightId) : byTitle;
|
||||
});
|
||||
}
|
||||
|
||||
function cloneState(state: PickerState, patch: Partial<PickerState>): PickerState {
|
||||
return { ...state, ...patch };
|
||||
}
|
||||
|
||||
function transientHint(text: string, now: number): PickerState['transientHint'] {
|
||||
return { text, expiresAt: now + TRANSIENT_HINT_DURATION_MS };
|
||||
}
|
||||
|
||||
export function clearExpiredTransientHint(state: PickerState, now = Date.now()): PickerState {
|
||||
if (!state.transientHint || state.transientHint.expiresAt > now) {
|
||||
return state;
|
||||
}
|
||||
return cloneState(state, { transientHint: null });
|
||||
}
|
||||
|
||||
function ancestorsOf(nodeId: string, byId: Map<string, NotionPickerNode>): string[] {
|
||||
const ancestors: string[] = [];
|
||||
let parentId = byId.get(nodeId)?.parentId ?? null;
|
||||
const seen = new Set<string>();
|
||||
while (parentId && !seen.has(parentId)) {
|
||||
ancestors.push(parentId);
|
||||
seen.add(parentId);
|
||||
parentId = byId.get(parentId)?.parentId ?? null;
|
||||
}
|
||||
return ancestors;
|
||||
}
|
||||
|
||||
function descendantsOf(nodeId: string, byId: Map<string, NotionPickerNode>): string[] {
|
||||
const result: string[] = [];
|
||||
const stack = [...(byId.get(nodeId)?.childIds ?? [])].reverse();
|
||||
while (stack.length > 0) {
|
||||
const id = stack.pop();
|
||||
if (!id) {
|
||||
continue;
|
||||
}
|
||||
result.push(id);
|
||||
const node = byId.get(id);
|
||||
if (node) {
|
||||
stack.push(...[...node.childIds].reverse());
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function matchingIds(state: PickerState): Set<string> {
|
||||
const query = state.search.query.trim().toLocaleLowerCase();
|
||||
if (!query) {
|
||||
return new Set(state.tree.map((node) => node.id));
|
||||
}
|
||||
return new Set(
|
||||
state.tree
|
||||
.filter((node) => {
|
||||
const title = node.title.toLocaleLowerCase();
|
||||
const path = node.path.toLocaleLowerCase();
|
||||
return title.includes(query) || path.includes(query);
|
||||
})
|
||||
.map((node) => node.id),
|
||||
);
|
||||
}
|
||||
|
||||
export function buildPickerTree(searchResults: NotionPickerPageInput[]): NotionPickerNode[] {
|
||||
const nodes = new Map<string, MutableNode>();
|
||||
for (const result of searchResults) {
|
||||
const id = normalizePageId(result.id);
|
||||
if (nodes.has(id)) {
|
||||
continue;
|
||||
}
|
||||
nodes.set(id, {
|
||||
id,
|
||||
title: titleValue(result.title),
|
||||
archived: result.archived === true,
|
||||
parentId: result.parentId ? normalizePageId(result.parentId) : null,
|
||||
childIds: [],
|
||||
});
|
||||
}
|
||||
|
||||
for (const node of nodes.values()) {
|
||||
if (!node.parentId || node.parentId === node.id || !nodes.has(node.parentId)) {
|
||||
node.parentId = null;
|
||||
continue;
|
||||
}
|
||||
|
||||
const seen = new Set([node.id]);
|
||||
let cursor: string | null = node.parentId;
|
||||
while (cursor) {
|
||||
if (seen.has(cursor)) {
|
||||
node.parentId = null;
|
||||
break;
|
||||
}
|
||||
seen.add(cursor);
|
||||
cursor = nodes.get(cursor)?.parentId ?? null;
|
||||
}
|
||||
}
|
||||
|
||||
for (const node of nodes.values()) {
|
||||
node.childIds = [];
|
||||
}
|
||||
for (const node of nodes.values()) {
|
||||
if (node.parentId) {
|
||||
nodes.get(node.parentId)?.childIds.push(node.id);
|
||||
}
|
||||
}
|
||||
for (const node of nodes.values()) {
|
||||
node.childIds = sortedNodeIds(node.childIds, nodes);
|
||||
}
|
||||
|
||||
const roots = sortedNodeIds(
|
||||
[...nodes.values()].filter((node) => node.parentId === null).map((node) => node.id),
|
||||
nodes,
|
||||
);
|
||||
const tree: NotionPickerNode[] = [];
|
||||
|
||||
function visit(nodeId: string, depth: number, pathPrefix: string[]): void {
|
||||
const raw = nodes.get(nodeId);
|
||||
if (!raw) {
|
||||
return;
|
||||
}
|
||||
const path = [...pathPrefix, raw.title].join(' / ');
|
||||
const node: NotionPickerNode = {
|
||||
id: raw.id,
|
||||
title: raw.title,
|
||||
archived: raw.archived,
|
||||
parentId: raw.parentId,
|
||||
depth,
|
||||
childIds: raw.childIds,
|
||||
path,
|
||||
};
|
||||
tree.push(node);
|
||||
for (const childId of raw.childIds) {
|
||||
visit(childId, depth + 1, [...pathPrefix, raw.title]);
|
||||
}
|
||||
}
|
||||
|
||||
for (const rootId of roots) {
|
||||
visit(rootId, 0, []);
|
||||
}
|
||||
|
||||
return tree;
|
||||
}
|
||||
|
||||
export function isAncestorChecked(nodeId: string, checked: Set<string>, byId: Map<string, NotionPickerNode>): boolean {
|
||||
return ancestorsOf(nodeId, byId).some((ancestorId) => checked.has(ancestorId));
|
||||
}
|
||||
|
||||
function checkedAncestor(nodeId: string, state: PickerState): NotionPickerNode | null {
|
||||
for (const ancestorId of ancestorsOf(nodeId, state.byId)) {
|
||||
if (state.checked.has(ancestorId)) {
|
||||
return state.byId.get(ancestorId) ?? null;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function canToggle(nodeId: string, state: PickerState): { ok: true } | { ok: false; reason: string } {
|
||||
if (!state.byId.has(nodeId)) {
|
||||
return { ok: false, reason: 'Page not found' };
|
||||
}
|
||||
const ancestor = checkedAncestor(nodeId, state);
|
||||
if (ancestor) {
|
||||
return { ok: false, reason: `Locked by '${ancestor.title}' - uncheck parent first` };
|
||||
}
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
export function toggleChecked(state: PickerState, nodeId: string, now = Date.now()): PickerState {
|
||||
const toggle = canToggle(nodeId, state);
|
||||
if (!toggle.ok) {
|
||||
return cloneState(state, {
|
||||
transientHint: transientHint(toggle.reason, now),
|
||||
});
|
||||
}
|
||||
|
||||
const checked = new Set(state.checked);
|
||||
if (checked.has(nodeId)) {
|
||||
checked.delete(nodeId);
|
||||
} else {
|
||||
checked.add(nodeId);
|
||||
for (const descendantId of descendantsOf(nodeId, state.byId)) {
|
||||
checked.delete(descendantId);
|
||||
}
|
||||
}
|
||||
return cloneState(state, { checked, transientHint: null });
|
||||
}
|
||||
|
||||
export function flattenSelection(checked: Set<string>, byId: Map<string, NotionPickerNode>): string[] {
|
||||
const result: string[] = [];
|
||||
for (const node of byId.values()) {
|
||||
if (checked.has(node.id) && !isAncestorChecked(node.id, checked, byId)) {
|
||||
result.push(node.id);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
export function filterTree(state: PickerState): { visibleIds: Set<string>; autoExpand: Set<string> } {
|
||||
const matches = matchingIds(state);
|
||||
if (state.search.query.trim().length === 0) {
|
||||
return { visibleIds: matches, autoExpand: new Set() };
|
||||
}
|
||||
|
||||
const visibleIds = new Set<string>();
|
||||
const autoExpand = new Set<string>();
|
||||
for (const matchId of matches) {
|
||||
visibleIds.add(matchId);
|
||||
for (const ancestorId of ancestorsOf(matchId, state.byId)) {
|
||||
visibleIds.add(ancestorId);
|
||||
autoExpand.add(ancestorId);
|
||||
}
|
||||
}
|
||||
return { visibleIds, autoExpand };
|
||||
}
|
||||
|
||||
export function visibleNodeIds(state: PickerState): string[] {
|
||||
const { visibleIds, autoExpand } = filterTree(state);
|
||||
const result: string[] = [];
|
||||
const roots = state.tree.filter((node) => node.parentId === null).map((node) => node.id);
|
||||
|
||||
function visit(nodeId: string): void {
|
||||
if (!visibleIds.has(nodeId)) {
|
||||
return;
|
||||
}
|
||||
result.push(nodeId);
|
||||
const node = state.byId.get(nodeId);
|
||||
if (!node) {
|
||||
return;
|
||||
}
|
||||
if (state.expanded.has(nodeId) || autoExpand.has(nodeId)) {
|
||||
for (const childId of node.childIds) {
|
||||
visit(childId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const rootId of roots) {
|
||||
visit(rootId);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
export function selectAllVisible(state: PickerState): PickerState {
|
||||
const candidates = state.search.query.trim().length > 0 ? matchingIds(state) : new Set(visibleNodeIds(state));
|
||||
const checked = new Set(state.checked);
|
||||
|
||||
for (const node of state.tree) {
|
||||
if (!candidates.has(node.id)) {
|
||||
continue;
|
||||
}
|
||||
const hasCandidateAncestor = ancestorsOf(node.id, state.byId).some((ancestorId) => candidates.has(ancestorId));
|
||||
if (!hasCandidateAncestor && !isAncestorChecked(node.id, checked, state.byId)) {
|
||||
checked.add(node.id);
|
||||
for (const descendantId of descendantsOf(node.id, state.byId)) {
|
||||
checked.delete(descendantId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return cloneState(state, {
|
||||
checked: new Set(flattenSelection(checked, state.byId)),
|
||||
transientHint: null,
|
||||
});
|
||||
}
|
||||
|
||||
export function selectNone(state: PickerState): PickerState {
|
||||
return cloneState(state, { checked: new Set(), transientHint: null });
|
||||
}
|
||||
|
||||
function setExpanded(state: PickerState, nodeId: string, value: boolean | 'toggle'): PickerState {
|
||||
const expanded = new Set(state.expanded);
|
||||
const nextValue = value === 'toggle' ? !expanded.has(nodeId) : value;
|
||||
if (nextValue) {
|
||||
expanded.add(nodeId);
|
||||
} else {
|
||||
expanded.delete(nodeId);
|
||||
}
|
||||
return cloneState(state, { expanded });
|
||||
}
|
||||
|
||||
function expandPath(state: PickerState, nodeId: string): PickerState {
|
||||
const expanded = new Set(state.expanded);
|
||||
for (const ancestorId of ancestorsOf(nodeId, state.byId)) {
|
||||
expanded.add(ancestorId);
|
||||
}
|
||||
return cloneState(state, { expanded });
|
||||
}
|
||||
|
||||
export function moveCursor(state: PickerState, dir: 'up' | 'down' | 'left' | 'right'): PickerState {
|
||||
const node = state.byId.get(state.cursorId);
|
||||
if (!node) {
|
||||
return state;
|
||||
}
|
||||
|
||||
if (dir === 'left') {
|
||||
if (node.childIds.length > 0 && state.expanded.has(node.id)) {
|
||||
return setExpanded(state, node.id, false);
|
||||
}
|
||||
return node.parentId ? cloneState(state, { cursorId: node.parentId }) : state;
|
||||
}
|
||||
|
||||
if (dir === 'right') {
|
||||
if (node.childIds.length === 0) {
|
||||
return state;
|
||||
}
|
||||
if (!state.expanded.has(node.id)) {
|
||||
return setExpanded(state, node.id, true);
|
||||
}
|
||||
return cloneState(state, { cursorId: node.childIds[0] ?? node.id });
|
||||
}
|
||||
|
||||
const ids = visibleNodeIds(state);
|
||||
const index = ids.indexOf(state.cursorId);
|
||||
if (index === -1) {
|
||||
return ids[0] ? cloneState(state, { cursorId: ids[0] }) : state;
|
||||
}
|
||||
const nextIndex = dir === 'up' ? Math.max(0, index - 1) : Math.min(ids.length - 1, index + 1);
|
||||
return cloneState(state, { cursorId: ids[nextIndex] ?? state.cursorId });
|
||||
}
|
||||
|
||||
export function buildInitialState(args: {
|
||||
tree: NotionPickerNode[];
|
||||
existingRootPageIds: string[];
|
||||
currentCrawlMode?: 'all_accessible' | 'selected_roots';
|
||||
}): PickerState {
|
||||
const byId = new Map(args.tree.map((node) => [node.id, node]));
|
||||
const checked = new Set<string>();
|
||||
let staleCount = 0;
|
||||
|
||||
for (const rawId of args.existingRootPageIds) {
|
||||
const id = normalizePageId(rawId);
|
||||
if (byId.has(id)) {
|
||||
checked.add(id);
|
||||
} else {
|
||||
staleCount += 1;
|
||||
}
|
||||
}
|
||||
|
||||
const minimalChecked = new Set(flattenSelection(checked, byId));
|
||||
const expanded = new Set<string>();
|
||||
for (const checkedId of minimalChecked) {
|
||||
for (const ancestorId of ancestorsOf(checkedId, byId)) {
|
||||
expanded.add(ancestorId);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
tree: args.tree,
|
||||
byId,
|
||||
expanded,
|
||||
checked: minimalChecked,
|
||||
cursorId: args.tree[0]?.id ?? '',
|
||||
search: { editing: false, query: '' },
|
||||
pendingConfirm: null,
|
||||
preLoadWarnings: staleCount > 0 ? [`${staleCount} stored root_page_ids no longer visible`] : [],
|
||||
transientHint: null,
|
||||
currentCrawlMode: args.currentCrawlMode ?? 'selected_roots',
|
||||
};
|
||||
}
|
||||
|
||||
export function reducer(state: PickerState, cmd: PickerCommand, now = Date.now()): { next: PickerState; effect: PickerEffect } {
|
||||
if (state.pendingConfirm) {
|
||||
if (cmd === 'save-confirm') {
|
||||
return { next: cloneState(state, { pendingConfirm: null }), effect: 'save' };
|
||||
}
|
||||
if (cmd === 'save-cancel') {
|
||||
return { next: cloneState(state, { pendingConfirm: null }), effect: null };
|
||||
}
|
||||
if (cmd === 'quit') {
|
||||
return { next: state, effect: 'quit-without-save' };
|
||||
}
|
||||
return { next: state, effect: null };
|
||||
}
|
||||
|
||||
switch (cmd) {
|
||||
case 'cursor-up':
|
||||
return { next: moveCursor(state, 'up'), effect: null };
|
||||
case 'cursor-down':
|
||||
return { next: moveCursor(state, 'down'), effect: null };
|
||||
case 'cursor-left':
|
||||
return { next: moveCursor(state, 'left'), effect: null };
|
||||
case 'cursor-right':
|
||||
return { next: moveCursor(state, 'right'), effect: null };
|
||||
case 'expand':
|
||||
return { next: setExpanded(state, state.cursorId, 'toggle'), effect: null };
|
||||
case 'collapse':
|
||||
return { next: setExpanded(state, state.cursorId, false), effect: null };
|
||||
case 'expand-all':
|
||||
return {
|
||||
next: cloneState(state, {
|
||||
expanded: new Set(state.tree.filter((node) => node.childIds.length > 0).map((node) => node.id)),
|
||||
}),
|
||||
effect: null,
|
||||
};
|
||||
case 'collapse-all':
|
||||
return { next: cloneState(state, { expanded: new Set() }), effect: null };
|
||||
case 'toggle-check':
|
||||
return { next: toggleChecked(state, state.cursorId, now), effect: null };
|
||||
case 'select-all-visible':
|
||||
return { next: selectAllVisible(state), effect: null };
|
||||
case 'select-none':
|
||||
return { next: selectNone(state), effect: null };
|
||||
case 'clear-transient-hint':
|
||||
return { next: clearExpiredTransientHint(state, now), effect: null };
|
||||
case 'search-start':
|
||||
return { next: cloneState(state, { search: { ...state.search, editing: true } }), effect: null };
|
||||
case 'search-cancel':
|
||||
return { next: cloneState(state, { search: { editing: false, query: '' } }), effect: null };
|
||||
case 'search-submit':
|
||||
return { next: cloneState(state, { search: { ...state.search, editing: false } }), effect: null };
|
||||
case 'search-backspace':
|
||||
return {
|
||||
next: cloneState(state, { search: { ...state.search, query: state.search.query.slice(0, -1) } }),
|
||||
effect: null,
|
||||
};
|
||||
case 'save-request':
|
||||
if (state.checked.size === 0) {
|
||||
return {
|
||||
next: cloneState(state, {
|
||||
transientHint: transientHint('Select at least one page or press q to quit', now),
|
||||
}),
|
||||
effect: null,
|
||||
};
|
||||
}
|
||||
if (state.currentCrawlMode === 'all_accessible') {
|
||||
return { next: cloneState(state, { pendingConfirm: 'mode-switch' }), effect: null };
|
||||
}
|
||||
return { next: state, effect: 'save' };
|
||||
case 'save-confirm':
|
||||
return { next: state, effect: 'save' };
|
||||
case 'save-cancel':
|
||||
return { next: state, effect: null };
|
||||
case 'quit':
|
||||
return { next: state, effect: 'quit-without-save' };
|
||||
default:
|
||||
return { next: cloneState(state, { search: { ...state.search, query: state.search.query + cmd.value } }), effect: null };
|
||||
}
|
||||
}
|
||||
384
packages/cli/src/commands/connection-notion-tui.test.tsx
Normal file
384
packages/cli/src/commands/connection-notion-tui.test.tsx
Normal file
|
|
@ -0,0 +1,384 @@
|
|||
/* @jsxImportSource react */
|
||||
import { render as renderInkTest } from 'ink-testing-library';
|
||||
import React, { act, type ReactNode } from 'react';
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||
import { buildInitialState, buildPickerTree, type NotionPickerPageInput } from './connection-notion-tree.js';
|
||||
import {
|
||||
NotionPickerApp,
|
||||
notionPickerCommandForInkInput,
|
||||
renderNotionPickerTui,
|
||||
resolveNotionPickerWidth,
|
||||
sanitizeNotionPickerTuiError,
|
||||
windowItems,
|
||||
windowOffset,
|
||||
type NotionPickerInkInstance,
|
||||
type NotionPickerInkRenderOptions,
|
||||
} from './connection-notion-tui.js';
|
||||
|
||||
const IDS = {
|
||||
engineering: '11111111-1111-1111-1111-111111111111',
|
||||
architecture: '22222222-2222-2222-2222-222222222222',
|
||||
marketing: '33333333-3333-3333-3333-333333333333',
|
||||
finance: '44444444-4444-4444-4444-444444444444',
|
||||
ops: '55555555-5555-5555-5555-555555555555',
|
||||
sales: '66666666-6666-6666-6666-666666666666',
|
||||
support: '77777777-7777-7777-7777-777777777777',
|
||||
product: '88888888-8888-8888-8888-888888888888',
|
||||
design: '99999999-9999-9999-9999-999999999999',
|
||||
};
|
||||
|
||||
function pages(): NotionPickerPageInput[] {
|
||||
return [
|
||||
{ id: IDS.engineering, title: 'Engineering Docs', archived: false, parentId: null },
|
||||
{ id: IDS.architecture, title: 'Architecture', archived: false, parentId: IDS.engineering },
|
||||
{ id: IDS.marketing, title: 'Marketing', archived: false, parentId: null },
|
||||
];
|
||||
}
|
||||
|
||||
function manyPages(): NotionPickerPageInput[] {
|
||||
return [
|
||||
{ id: IDS.engineering, title: 'Engineering Docs', archived: false, parentId: null },
|
||||
{ id: IDS.architecture, title: 'Architecture', archived: false, parentId: IDS.engineering },
|
||||
{ id: IDS.marketing, title: 'Marketing', archived: false, parentId: null },
|
||||
{ id: IDS.finance, title: 'Finance', archived: false, parentId: null },
|
||||
{ id: IDS.ops, title: 'Operations', archived: false, parentId: null },
|
||||
{ id: IDS.sales, title: 'Sales', archived: false, parentId: null },
|
||||
{ id: IDS.support, title: 'Support', archived: false, parentId: null },
|
||||
{ id: IDS.product, title: 'Product', archived: false, parentId: null },
|
||||
{ id: IDS.design, title: 'Design', archived: false, parentId: null },
|
||||
];
|
||||
}
|
||||
|
||||
function state(mode: 'all_accessible' | 'selected_roots' = 'selected_roots') {
|
||||
return buildInitialState({
|
||||
tree: buildPickerTree(pages()),
|
||||
existingRootPageIds: [],
|
||||
currentCrawlMode: mode,
|
||||
});
|
||||
}
|
||||
|
||||
async function waitForInkInput(): Promise<void> {
|
||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||
}
|
||||
|
||||
function fakeInkInstance(): NotionPickerInkInstance {
|
||||
return {
|
||||
rerender: vi.fn(),
|
||||
unmount: vi.fn(),
|
||||
waitUntilExit: vi.fn(async () => undefined),
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeFrameWrap(frame: string | undefined): string {
|
||||
return frame?.replace(/\n/g, ' ') ?? '';
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
describe('notionPickerCommandForInkInput', () => {
|
||||
it('maps browse, search, and confirm input to reducer commands', () => {
|
||||
expect(notionPickerCommandForInkInput('', { downArrow: true }, state().search, null)).toBe('cursor-down');
|
||||
expect(notionPickerCommandForInkInput('', { upArrow: true }, state().search, null)).toBe('cursor-up');
|
||||
expect(notionPickerCommandForInkInput('', { rightArrow: true }, state().search, null)).toBe('cursor-right');
|
||||
expect(notionPickerCommandForInkInput('', { leftArrow: true }, state().search, null)).toBe('cursor-left');
|
||||
expect(notionPickerCommandForInkInput(' ', {}, state().search, null)).toBe('toggle-check');
|
||||
expect(notionPickerCommandForInkInput('/', {}, state().search, null)).toBe('search-start');
|
||||
expect(notionPickerCommandForInkInput('a', {}, state().search, null)).toBe('select-all-visible');
|
||||
expect(notionPickerCommandForInkInput('n', {}, state().search, null)).toBe('select-none');
|
||||
expect(notionPickerCommandForInkInput('s', {}, state().search, null)).toBe('save-request');
|
||||
expect(notionPickerCommandForInkInput('q', {}, state().search, null)).toBe('quit');
|
||||
expect(notionPickerCommandForInkInput('c', { ctrl: true }, state().search, null)).toBe('quit');
|
||||
|
||||
expect(notionPickerCommandForInkInput('x', {}, { editing: true, query: '' }, null)).toEqual({
|
||||
type: 'search-input',
|
||||
value: 'x',
|
||||
});
|
||||
expect(notionPickerCommandForInkInput('', { backspace: true }, { editing: true, query: 'x' }, null)).toBe(
|
||||
'search-backspace',
|
||||
);
|
||||
expect(notionPickerCommandForInkInput('', { return: true }, { editing: true, query: 'x' }, null)).toBe(
|
||||
'search-submit',
|
||||
);
|
||||
expect(notionPickerCommandForInkInput('', { escape: true }, { editing: true, query: 'x' }, null)).toBe(
|
||||
'search-cancel',
|
||||
);
|
||||
|
||||
expect(notionPickerCommandForInkInput('y', {}, state().search, 'mode-switch')).toBe('save-confirm');
|
||||
expect(notionPickerCommandForInkInput('', { return: true }, state().search, 'mode-switch')).toBe('save-confirm');
|
||||
expect(notionPickerCommandForInkInput('n', {}, state().search, 'mode-switch')).toBe('save-cancel');
|
||||
});
|
||||
});
|
||||
|
||||
describe('window helpers', () => {
|
||||
it('centers the selected row and returns the visible slice', () => {
|
||||
expect(windowOffset(20, 10, 5)).toBe(8);
|
||||
expect(windowItems(['a', 'b', 'c', 'd', 'e'], 3, 3)).toEqual({ items: ['c', 'd', 'e'], offset: 2 });
|
||||
});
|
||||
|
||||
it('clamps picker width to the design rule', () => {
|
||||
expect(resolveNotionPickerWidth(200)).toBe(120);
|
||||
expect(resolveNotionPickerWidth(100)).toBe(96);
|
||||
expect(resolveNotionPickerWidth(50)).toBe(60);
|
||||
expect(resolveNotionPickerWidth(undefined)).toBe(96);
|
||||
});
|
||||
});
|
||||
|
||||
describe('NotionPickerApp', () => {
|
||||
it('renders spec banners, row glyphs, search visibility, and hint text', () => {
|
||||
const initialState = {
|
||||
...state('all_accessible'),
|
||||
preLoadWarnings: ['1 stored root_page_ids no longer visible'],
|
||||
};
|
||||
const { lastFrame } = renderInkTest(
|
||||
<NotionPickerApp
|
||||
initialState={initialState}
|
||||
connectionId="notion-main"
|
||||
workspaceLabel="Design Workspace"
|
||||
cappedAtCount={5000}
|
||||
currentCrawlMode="all_accessible"
|
||||
terminalRows={24}
|
||||
terminalWidth={100}
|
||||
onExit={vi.fn()}
|
||||
/>,
|
||||
);
|
||||
|
||||
const frame = lastFrame() ?? '';
|
||||
expect(frame).toContain('Notion pages visible to integration "Design Workspace"');
|
||||
expect(frame).toContain('5000-page cap reached - some pages not shown');
|
||||
expect(frame).toContain('1 stored root_page_ids no longer visible - they will be removed if you save');
|
||||
expect(frame).toContain('▸ [ ] Engineering Docs ▸ (1)');
|
||||
expect(frame).toContain(' [ ] Marketing');
|
||||
expect(frame).not.toContain('Search ready: -');
|
||||
expect(frame).toContain('space toggle · enter expand · / search · a all · n none · s save & exit · q quit');
|
||||
});
|
||||
|
||||
it('renders partial discovery warnings without stale-root save suffix', () => {
|
||||
const initialState = {
|
||||
...state(),
|
||||
preLoadWarnings: ['Notion search stopped early: rate limit after first page'],
|
||||
};
|
||||
const { lastFrame } = renderInkTest(
|
||||
<NotionPickerApp
|
||||
initialState={initialState}
|
||||
connectionId="notion-main"
|
||||
workspaceLabel="Design Workspace"
|
||||
cappedAtCount={null}
|
||||
currentCrawlMode="selected_roots"
|
||||
terminalRows={24}
|
||||
terminalWidth={100}
|
||||
onExit={vi.fn()}
|
||||
/>,
|
||||
);
|
||||
|
||||
const frame = lastFrame() ?? '';
|
||||
expect(frame).toContain('Notion search stopped early: rate limit after first page');
|
||||
expect(frame).not.toContain(
|
||||
'Notion search stopped early: rate limit after first page - they will be removed if you save',
|
||||
);
|
||||
});
|
||||
|
||||
it('renders checked parents and locked descendants with the locked design glyphs', () => {
|
||||
const initialState = {
|
||||
...state(),
|
||||
checked: new Set([IDS.engineering]),
|
||||
expanded: new Set([IDS.engineering]),
|
||||
};
|
||||
const { lastFrame } = renderInkTest(
|
||||
<NotionPickerApp
|
||||
initialState={initialState}
|
||||
connectionId="notion-main"
|
||||
workspaceLabel="Design Workspace"
|
||||
cappedAtCount={null}
|
||||
currentCrawlMode="selected_roots"
|
||||
terminalRows={24}
|
||||
terminalWidth={100}
|
||||
onExit={vi.fn()}
|
||||
/>,
|
||||
);
|
||||
|
||||
const frame = lastFrame() ?? '';
|
||||
expect(frame).toContain('▸ [×] Engineering Docs ▾');
|
||||
expect(frame).toContain(' [~] Architecture');
|
||||
});
|
||||
|
||||
it('supports keyboard selection, all_accessible confirmation, and save callback', async () => {
|
||||
const onExit = vi.fn();
|
||||
const { stdin, lastFrame } = renderInkTest(
|
||||
<NotionPickerApp
|
||||
initialState={state('all_accessible')}
|
||||
connectionId="notion-main"
|
||||
workspaceLabel="Design Workspace"
|
||||
cappedAtCount={null}
|
||||
currentCrawlMode="all_accessible"
|
||||
terminalRows={24}
|
||||
terminalWidth={100}
|
||||
onExit={onExit}
|
||||
/>,
|
||||
);
|
||||
|
||||
stdin.write(' ');
|
||||
await waitForInkInput();
|
||||
expect(lastFrame()).toContain('[×] Engineering Docs');
|
||||
|
||||
stdin.write('s');
|
||||
await waitForInkInput();
|
||||
expect(normalizeFrameWrap(lastFrame())).toContain(
|
||||
'Save will switch crawl_mode all_accessible -> selected_roots and limit ingest to 1 selected page. [y] confirm [esc] back',
|
||||
);
|
||||
|
||||
stdin.write('y');
|
||||
await waitForInkInput();
|
||||
expect(onExit).toHaveBeenCalledWith({ kind: 'save', rootPageIds: [IDS.engineering] });
|
||||
});
|
||||
|
||||
it('removes transient hints after their expiry time', async () => {
|
||||
vi.useFakeTimers();
|
||||
const onExit = vi.fn();
|
||||
const { stdin, lastFrame } = renderInkTest(
|
||||
<NotionPickerApp
|
||||
initialState={state()}
|
||||
connectionId="notion-main"
|
||||
workspaceLabel="Design Workspace"
|
||||
cappedAtCount={null}
|
||||
currentCrawlMode="selected_roots"
|
||||
terminalRows={24}
|
||||
terminalWidth={100}
|
||||
onExit={onExit}
|
||||
/>,
|
||||
);
|
||||
|
||||
await act(async () => {
|
||||
stdin.write('s');
|
||||
await vi.advanceTimersByTimeAsync(10);
|
||||
});
|
||||
expect(lastFrame()).toContain('Select at least one page or press q to quit');
|
||||
|
||||
await act(async () => {
|
||||
await vi.advanceTimersByTimeAsync(2500);
|
||||
});
|
||||
expect(lastFrame()).not.toContain('Select at least one page or press q to quit');
|
||||
expect(onExit).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('renders row-window overflow indicators when the visible list is clipped', async () => {
|
||||
const onExit = vi.fn();
|
||||
const initialState = buildInitialState({
|
||||
tree: buildPickerTree(manyPages()),
|
||||
existingRootPageIds: [],
|
||||
currentCrawlMode: 'selected_roots',
|
||||
});
|
||||
initialState.expanded = new Set([IDS.engineering]);
|
||||
const { stdin, lastFrame } = renderInkTest(
|
||||
<NotionPickerApp
|
||||
initialState={initialState}
|
||||
connectionId="notion-main"
|
||||
workspaceLabel="Design Workspace"
|
||||
cappedAtCount={null}
|
||||
currentCrawlMode="selected_roots"
|
||||
terminalRows={13}
|
||||
terminalWidth={100}
|
||||
onExit={onExit}
|
||||
/>,
|
||||
);
|
||||
|
||||
expect(lastFrame()).toContain('↓ 4 more');
|
||||
|
||||
stdin.write('\u001B[B');
|
||||
stdin.write('\u001B[B');
|
||||
stdin.write('\u001B[B');
|
||||
stdin.write('\u001B[B');
|
||||
await waitForInkInput();
|
||||
|
||||
const frame = lastFrame() ?? '';
|
||||
expect(frame).toContain('↑ ');
|
||||
expect(frame).toContain('↓ ');
|
||||
expect(onExit).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('returns quit without saving', async () => {
|
||||
const onExit = vi.fn();
|
||||
const { stdin } = renderInkTest(
|
||||
<NotionPickerApp
|
||||
initialState={state()}
|
||||
connectionId="notion-main"
|
||||
workspaceLabel="Design Workspace"
|
||||
cappedAtCount={null}
|
||||
currentCrawlMode="selected_roots"
|
||||
terminalRows={24}
|
||||
terminalWidth={100}
|
||||
onExit={onExit}
|
||||
/>,
|
||||
);
|
||||
|
||||
stdin.write('q');
|
||||
await waitForInkInput();
|
||||
expect(onExit).toHaveBeenCalledWith({ kind: 'quit' });
|
||||
});
|
||||
});
|
||||
|
||||
describe('renderNotionPickerTui', () => {
|
||||
it('returns the app result from the Ink runtime', async () => {
|
||||
const io = {
|
||||
stdin: { isTTY: true, setRawMode: vi.fn() },
|
||||
stdout: { isTTY: true, columns: 100, rows: 24, write: vi.fn() },
|
||||
stderr: { write: vi.fn() },
|
||||
};
|
||||
const renderInk = vi.fn((_tree: ReactNode, _options: NotionPickerInkRenderOptions) => fakeInkInstance());
|
||||
|
||||
await expect(
|
||||
renderNotionPickerTui(
|
||||
{
|
||||
initialState: state(),
|
||||
connectionId: 'notion-main',
|
||||
workspaceLabel: 'Design Workspace',
|
||||
cappedAtCount: null,
|
||||
currentCrawlMode: 'selected_roots',
|
||||
},
|
||||
io,
|
||||
{ renderInk },
|
||||
),
|
||||
).resolves.toEqual({ kind: 'quit' });
|
||||
expect(renderInk).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it('sanitizes render errors and tells the user to use no-input mode', async () => {
|
||||
expect(sanitizeNotionPickerTuiError(new Error('token=secret https://api.notion.com/v1/search'))).toBe(
|
||||
'[redacted] [redacted-url]',
|
||||
);
|
||||
});
|
||||
|
||||
it('falls back to quit with a scripted-mode hint when Ink cannot initialize', async () => {
|
||||
let stderr = '';
|
||||
const io = {
|
||||
stdin: { isTTY: false, setRawMode: vi.fn() },
|
||||
stdout: { isTTY: false, columns: 100, rows: 24, write: vi.fn() },
|
||||
stderr: {
|
||||
write(chunk: string) {
|
||||
stderr += chunk;
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
await expect(
|
||||
renderNotionPickerTui(
|
||||
{
|
||||
initialState: state(),
|
||||
connectionId: 'notion-main',
|
||||
workspaceLabel: 'Design Workspace',
|
||||
cappedAtCount: null,
|
||||
currentCrawlMode: 'selected_roots',
|
||||
},
|
||||
io,
|
||||
{
|
||||
renderInk: vi.fn(() => {
|
||||
throw new Error('token=secret');
|
||||
}),
|
||||
},
|
||||
),
|
||||
).resolves.toEqual({ kind: 'quit' });
|
||||
expect(stderr).toContain('Use --no-input --root-page-id <UUID> for scripted mode');
|
||||
expect(stderr).not.toContain('secret');
|
||||
});
|
||||
});
|
||||
338
packages/cli/src/commands/connection-notion-tui.tsx
Normal file
338
packages/cli/src/commands/connection-notion-tui.tsx
Normal file
|
|
@ -0,0 +1,338 @@
|
|||
/* @jsxImportSource react */
|
||||
import { Box, Text, render as renderInkRuntime, useApp, useInput } from 'ink';
|
||||
import React, { type ReactNode, useEffect, useMemo, useRef, useState } from 'react';
|
||||
import {
|
||||
filterTree,
|
||||
flattenSelection,
|
||||
isAncestorChecked,
|
||||
reducer,
|
||||
visibleNodeIds,
|
||||
type PickerCommand,
|
||||
type PickerState,
|
||||
} from './connection-notion-tree.js';
|
||||
import type { KloCliIo } from '../index.js';
|
||||
|
||||
const COLOR_THEME = {
|
||||
text: 'white',
|
||||
muted: 'gray',
|
||||
active: 'cyan',
|
||||
warning: 'yellow',
|
||||
} as const;
|
||||
|
||||
const NO_COLOR_THEME = {
|
||||
text: 'white',
|
||||
muted: 'white',
|
||||
active: 'white',
|
||||
warning: 'white',
|
||||
} as const;
|
||||
|
||||
type NotionPickerTheme = Record<keyof typeof COLOR_THEME, string>;
|
||||
|
||||
export interface NotionPickerTuiIo extends KloCliIo {
|
||||
stdin?: { isTTY?: boolean; setRawMode?(value: boolean): void };
|
||||
stdout: KloCliIo['stdout'] & { isTTY?: boolean; columns?: number; rows?: number };
|
||||
}
|
||||
|
||||
interface InkKey {
|
||||
leftArrow?: boolean;
|
||||
rightArrow?: boolean;
|
||||
upArrow?: boolean;
|
||||
downArrow?: boolean;
|
||||
return?: boolean;
|
||||
escape?: boolean;
|
||||
ctrl?: boolean;
|
||||
backspace?: boolean;
|
||||
delete?: boolean;
|
||||
}
|
||||
|
||||
export type PickerRenderResult = { kind: 'save'; rootPageIds: string[] } | { kind: 'quit' };
|
||||
|
||||
export interface PickerRenderInput {
|
||||
initialState: PickerState;
|
||||
connectionId: string;
|
||||
workspaceLabel: string;
|
||||
cappedAtCount: number | null;
|
||||
currentCrawlMode: 'all_accessible' | 'selected_roots';
|
||||
}
|
||||
|
||||
interface NotionPickerAppProps extends PickerRenderInput {
|
||||
terminalRows?: number;
|
||||
terminalWidth?: number;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
onExit(result: PickerRenderResult): void;
|
||||
}
|
||||
|
||||
export interface NotionPickerInkInstance {
|
||||
rerender(tree: ReactNode): void;
|
||||
unmount(): void;
|
||||
waitUntilExit(): Promise<void>;
|
||||
}
|
||||
|
||||
export interface NotionPickerInkRenderOptions {
|
||||
stdin?: NotionPickerTuiIo['stdin'];
|
||||
stdout: NotionPickerTuiIo['stdout'];
|
||||
stderr: NotionPickerTuiIo['stderr'];
|
||||
exitOnCtrlC: boolean;
|
||||
patchConsole: boolean;
|
||||
maxFps: number;
|
||||
alternateScreen: boolean;
|
||||
}
|
||||
|
||||
function resolveTheme(env: NodeJS.ProcessEnv = process.env): NotionPickerTheme {
|
||||
return env.NO_COLOR || env.TERM === 'dumb' ? NO_COLOR_THEME : COLOR_THEME;
|
||||
}
|
||||
|
||||
export function resolveNotionPickerWidth(columns: number | undefined): number {
|
||||
const resolvedColumns = columns ?? 100;
|
||||
return Math.max(60, Math.min(120, resolvedColumns - 4));
|
||||
}
|
||||
|
||||
function staleWarningText(warning: string): string {
|
||||
return warning.includes('stored root_page_ids no longer visible')
|
||||
? `${warning} - they will be removed if you save`
|
||||
: warning;
|
||||
}
|
||||
|
||||
function selectedPageCountText(count: number): string {
|
||||
return `${count} selected ${count === 1 ? 'page' : 'pages'}`;
|
||||
}
|
||||
|
||||
function rowMatchesSearch(state: PickerState, nodeId: string): boolean {
|
||||
const query = state.search.query.trim().toLocaleLowerCase();
|
||||
if (!query) {
|
||||
return false;
|
||||
}
|
||||
const node = state.byId.get(nodeId);
|
||||
if (!node) {
|
||||
return false;
|
||||
}
|
||||
return node.title.toLocaleLowerCase().includes(query) || node.path.toLocaleLowerCase().includes(query);
|
||||
}
|
||||
|
||||
export function sanitizeNotionPickerTuiError(error: unknown): string {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
return message
|
||||
.replace(/[a-z][a-z0-9+.-]*:\/\/[^\s]+/gi, '[redacted-url]')
|
||||
.replace(/\b(api[_-]?key|password|token|secret)=\S+/gi, '[redacted]');
|
||||
}
|
||||
|
||||
export function windowOffset(count: number, selected: number, visible: number): number {
|
||||
if (count <= visible) return 0;
|
||||
return Math.max(0, Math.min(count - visible, selected - Math.floor(visible / 2)));
|
||||
}
|
||||
|
||||
export function windowItems<T>(items: T[], selected: number, visible: number): { items: T[]; offset: number } {
|
||||
const offset = windowOffset(items.length, selected, visible);
|
||||
return { items: items.slice(offset, offset + visible), offset };
|
||||
}
|
||||
|
||||
function truncateText(value: string, width: number): string {
|
||||
if (value.length <= width) return value;
|
||||
if (width <= 3) return value.slice(0, width);
|
||||
return `${value.slice(0, width - 3)}...`;
|
||||
}
|
||||
|
||||
export function notionPickerCommandForInkInput(
|
||||
input: string,
|
||||
key: InkKey,
|
||||
search: PickerState['search'],
|
||||
pendingConfirm: PickerState['pendingConfirm'],
|
||||
): PickerCommand | null {
|
||||
if (pendingConfirm) {
|
||||
if (input === 'y' || key.return) return 'save-confirm';
|
||||
if (input === 'n' || key.escape) return 'save-cancel';
|
||||
if (key.ctrl === true && input === 'c') return 'quit';
|
||||
return null;
|
||||
}
|
||||
if (search.editing) {
|
||||
if (key.escape) return 'search-cancel';
|
||||
if (key.return) return 'search-submit';
|
||||
if (key.backspace || key.delete) return 'search-backspace';
|
||||
if (key.downArrow) return 'cursor-down';
|
||||
if (key.upArrow) return 'cursor-up';
|
||||
if (input.length === 1 && input >= ' ' && input !== '\u007f') return { type: 'search-input', value: input };
|
||||
return null;
|
||||
}
|
||||
if (key.ctrl === true && input === 'c') return 'quit';
|
||||
if (key.upArrow) return 'cursor-up';
|
||||
if (key.downArrow) return 'cursor-down';
|
||||
if (key.leftArrow) return 'cursor-left';
|
||||
if (key.rightArrow) return 'cursor-right';
|
||||
if (key.return) return 'expand';
|
||||
if (input === ' ') return 'toggle-check';
|
||||
if (input === '/') return 'search-start';
|
||||
if (input === 'a') return 'select-all-visible';
|
||||
if (input === 'n') return 'select-none';
|
||||
if (input === 's') return 'save-request';
|
||||
if (input === 'q' || key.escape) return 'quit';
|
||||
return null;
|
||||
}
|
||||
|
||||
function PickerRow(props: { state: PickerState; nodeId: string; width: number; theme: NotionPickerTheme }): ReactNode {
|
||||
const node = props.state.byId.get(props.nodeId);
|
||||
if (!node) return null;
|
||||
const focused = props.state.cursorId === node.id;
|
||||
const locked = isAncestorChecked(node.id, props.state.checked, props.state.byId);
|
||||
const checked = props.state.checked.has(node.id);
|
||||
const glyph = locked ? '[~]' : checked ? '[×]' : '[ ]';
|
||||
const children =
|
||||
node.childIds.length > 0 ? (props.state.expanded.has(node.id) ? ' ▾' : ` ▸ (${node.childIds.length})`) : '';
|
||||
const prefix = `${focused ? '▸' : ' '} ${glyph} ${' '.repeat(node.depth * 2)}`;
|
||||
const color = focused ? props.theme.active : locked || node.archived ? props.theme.muted : props.theme.text;
|
||||
const title = truncateText(`${node.title}${children}`, Math.max(10, props.width - prefix.length));
|
||||
const inverse = rowMatchesSearch(props.state, node.id);
|
||||
|
||||
return (
|
||||
<Text color={color} strikethrough={node.archived}>
|
||||
{prefix}
|
||||
<Text inverse={inverse}>{title}</Text>
|
||||
</Text>
|
||||
);
|
||||
}
|
||||
|
||||
export function NotionPickerApp(props: NotionPickerAppProps): ReactNode {
|
||||
const app = useApp();
|
||||
const [state, setState] = useState(props.initialState);
|
||||
const stateRef = useRef(state);
|
||||
const theme = useMemo(() => resolveTheme(props.env), [props.env]);
|
||||
const visibleIds = visibleNodeIds(state);
|
||||
const selectedIndex = Math.max(0, visibleIds.indexOf(state.cursorId));
|
||||
const reservedRows = state.pendingConfirm === 'mode-switch' ? 9 : 8;
|
||||
const visibleRows = Math.max(5, Math.min(20, (props.terminalRows ?? 24) - reservedRows));
|
||||
const rows = windowItems(visibleIds, selectedIndex, visibleRows);
|
||||
const hiddenAbove = rows.offset;
|
||||
const hiddenBelow = Math.max(0, visibleIds.length - rows.offset - rows.items.length);
|
||||
const searchMatchCount = filterTree(state).visibleIds.size;
|
||||
const width = resolveNotionPickerWidth(props.terminalWidth);
|
||||
const showSearch = state.search.editing || state.search.query.trim().length > 0;
|
||||
const selectedCount = flattenSelection(state.checked, state.byId).length;
|
||||
|
||||
stateRef.current = state;
|
||||
|
||||
useEffect(() => {
|
||||
const hint = state.transientHint;
|
||||
if (!hint) {
|
||||
return;
|
||||
}
|
||||
|
||||
const clearHint = () => {
|
||||
setState((current) => {
|
||||
const { next } = reducer(current, 'clear-transient-hint');
|
||||
stateRef.current = next;
|
||||
return next;
|
||||
});
|
||||
};
|
||||
const delay = hint.expiresAt - Date.now();
|
||||
if (delay <= 0) {
|
||||
clearHint();
|
||||
return;
|
||||
}
|
||||
|
||||
const timeout = setTimeout(clearHint, delay);
|
||||
|
||||
return () => clearTimeout(timeout);
|
||||
}, [state.transientHint?.expiresAt]);
|
||||
|
||||
useInput((input, key) => {
|
||||
const command = notionPickerCommandForInkInput(input, key, stateRef.current.search, stateRef.current.pendingConfirm);
|
||||
if (!command) {
|
||||
return;
|
||||
}
|
||||
const { next, effect } = reducer(stateRef.current, command);
|
||||
stateRef.current = next;
|
||||
setState(next);
|
||||
if (effect === 'save') {
|
||||
props.onExit({ kind: 'save', rootPageIds: flattenSelection(next.checked, next.byId) });
|
||||
app.exit();
|
||||
return;
|
||||
}
|
||||
if (effect === 'quit-without-save') {
|
||||
props.onExit({ kind: 'quit' });
|
||||
app.exit();
|
||||
}
|
||||
});
|
||||
|
||||
return (
|
||||
<Box flexDirection="column">
|
||||
<Text color={theme.active}>Notion pages visible to integration "{props.workspaceLabel}"</Text>
|
||||
{props.cappedAtCount ? <Text color={theme.warning}>{props.cappedAtCount}-page cap reached - some pages not shown</Text> : null}
|
||||
{state.preLoadWarnings.map((warning) => (
|
||||
<Text key={warning} color={theme.warning}>
|
||||
{staleWarningText(warning)}
|
||||
</Text>
|
||||
))}
|
||||
{showSearch ? (
|
||||
<Text color={theme.muted}>
|
||||
/ {state.search.query}
|
||||
{state.search.editing ? '█' : ''} ({searchMatchCount} matches)
|
||||
</Text>
|
||||
) : null}
|
||||
<Box flexDirection="column">
|
||||
{hiddenAbove > 0 ? <Text color={theme.muted}>↑ {hiddenAbove} more</Text> : null}
|
||||
{rows.items.map((nodeId) => (
|
||||
<PickerRow key={nodeId} state={state} nodeId={nodeId} width={width} theme={theme} />
|
||||
))}
|
||||
{hiddenBelow > 0 ? <Text color={theme.muted}>↓ {hiddenBelow} more</Text> : null}
|
||||
</Box>
|
||||
{state.pendingConfirm === 'mode-switch' ? (
|
||||
<Text color={theme.warning}>
|
||||
Save will switch crawl_mode all_accessible -> selected_roots and limit ingest to{' '}
|
||||
{selectedPageCountText(selectedCount)}. [y] confirm [esc] back
|
||||
</Text>
|
||||
) : null}
|
||||
{state.transientHint ? <Text color={theme.warning}>{state.transientHint.text}</Text> : null}
|
||||
<Text color={theme.muted}>space toggle · enter expand · / search · a all · n none · s save & exit · q quit</Text>
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
|
||||
function renderInk(tree: ReactNode, options: NotionPickerInkRenderOptions): NotionPickerInkInstance {
|
||||
return renderInkRuntime(tree, {
|
||||
stdin: options.stdin as NodeJS.ReadStream | undefined,
|
||||
stdout: options.stdout as NodeJS.WriteStream,
|
||||
stderr: options.stderr as NodeJS.WriteStream,
|
||||
exitOnCtrlC: options.exitOnCtrlC,
|
||||
patchConsole: options.patchConsole,
|
||||
maxFps: options.maxFps,
|
||||
alternateScreen: options.alternateScreen,
|
||||
}) as NotionPickerInkInstance;
|
||||
}
|
||||
|
||||
export async function renderNotionPickerTui(
|
||||
input: PickerRenderInput,
|
||||
io: NotionPickerTuiIo,
|
||||
options: { renderInk?: (tree: ReactNode, options: NotionPickerInkRenderOptions) => NotionPickerInkInstance } = {},
|
||||
): Promise<PickerRenderResult> {
|
||||
let result: PickerRenderResult = { kind: 'quit' };
|
||||
let instance: NotionPickerInkInstance | null = null;
|
||||
try {
|
||||
instance = (options.renderInk ?? renderInk)(
|
||||
<NotionPickerApp
|
||||
{...input}
|
||||
terminalRows={(io.stdout as { rows?: number }).rows ?? process.stdout.rows ?? 24}
|
||||
terminalWidth={io.stdout.columns ?? process.stdout.columns}
|
||||
onExit={(next) => {
|
||||
result = next;
|
||||
instance?.unmount();
|
||||
}}
|
||||
/>,
|
||||
{
|
||||
stdin: io.stdin,
|
||||
stdout: io.stdout,
|
||||
stderr: io.stderr,
|
||||
exitOnCtrlC: false,
|
||||
patchConsole: false,
|
||||
maxFps: 30,
|
||||
alternateScreen: true,
|
||||
},
|
||||
);
|
||||
await instance.waitUntilExit();
|
||||
instance.unmount();
|
||||
return result;
|
||||
} catch (error) {
|
||||
io.stderr.write(
|
||||
`Notion picker requires a TTY. Use --no-input --root-page-id <UUID> for scripted mode. ${sanitizeNotionPickerTuiError(error)}\n`,
|
||||
);
|
||||
return { kind: 'quit' };
|
||||
}
|
||||
}
|
||||
466
packages/cli/src/commands/connection-notion.test.ts
Normal file
466
packages/cli/src/commands/connection-notion.test.ts
Normal file
|
|
@ -0,0 +1,466 @@
|
|||
import { mkdtemp, readFile, rm } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import {
|
||||
initKloProject,
|
||||
loadKloProject,
|
||||
serializeKloProjectConfig,
|
||||
type KloProjectConfig,
|
||||
} from '@klo/context/project';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import {
|
||||
applyNotionPickerWriteback,
|
||||
discoverNotionPickerPages,
|
||||
notionPickerPageFromSearchResult,
|
||||
normalizeNotionPageId,
|
||||
resolveNotionWorkspaceLabel,
|
||||
runKloConnectionNotion,
|
||||
type NotionPickerApi,
|
||||
type PickerRenderInput,
|
||||
type PickerRenderResult,
|
||||
} from './connection-notion.js';
|
||||
|
||||
function makeIo() {
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
return {
|
||||
io: {
|
||||
stdout: {
|
||||
write: (chunk: string) => {
|
||||
stdout += chunk;
|
||||
},
|
||||
},
|
||||
stderr: {
|
||||
write: (chunk: string) => {
|
||||
stderr += chunk;
|
||||
},
|
||||
},
|
||||
},
|
||||
stdout: () => stdout,
|
||||
stderr: () => stderr,
|
||||
};
|
||||
}
|
||||
|
||||
type FakeNotionSearchPage = Record<string, unknown> & { id: string; object: 'page' };
|
||||
|
||||
const PAGE_IDS = {
|
||||
engineering: '11111111-1111-1111-1111-111111111111',
|
||||
architecture: '22222222-2222-2222-2222-222222222222',
|
||||
stale: '99999999-9999-9999-9999-999999999999',
|
||||
};
|
||||
|
||||
function notionPage(id: string, title: string, parentId: string | null = null): FakeNotionSearchPage {
|
||||
return {
|
||||
object: 'page',
|
||||
id,
|
||||
archived: false,
|
||||
parent: parentId ? { type: 'page_id', page_id: parentId } : { type: 'workspace', workspace: true },
|
||||
properties: {
|
||||
title: {
|
||||
type: 'title',
|
||||
title: [{ plain_text: title }],
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function fakeNotionApi(pages: FakeNotionSearchPage[]): NotionPickerApi {
|
||||
return {
|
||||
search: vi.fn(async (_filterValue, startCursor) => {
|
||||
if (startCursor === 'page-2') {
|
||||
return { results: pages.slice(2), hasMore: false, nextCursor: null };
|
||||
}
|
||||
return {
|
||||
results: pages.slice(0, 2),
|
||||
hasMore: pages.length > 2,
|
||||
nextCursor: pages.length > 2 ? 'page-2' : null,
|
||||
};
|
||||
}),
|
||||
retrieveBotUser: vi.fn(async () => ({ name: 'Notion bot', bot: { workspace_name: 'Design Workspace' } })),
|
||||
};
|
||||
}
|
||||
|
||||
describe('normalizeNotionPageId', () => {
|
||||
it('accepts dashed and compact UUIDs', () => {
|
||||
expect(normalizeNotionPageId('11111111222233334444555555555555')).toBe(
|
||||
'11111111-2222-3333-4444-555555555555',
|
||||
);
|
||||
expect(normalizeNotionPageId('AAAAAAAA-BBBB-CCCC-DDDD-EEEEEEEEEEEE')).toBe(
|
||||
'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('runKloConnectionNotion', () => {
|
||||
let tempDir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'klo-cli-notion-pick-'));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(tempDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
async function writeProjectConfig(projectDir: string, config: KloProjectConfig): Promise<void> {
|
||||
const project = await loadKloProject({ projectDir });
|
||||
await project.fileStore.writeFile(
|
||||
'klo.yaml',
|
||||
serializeKloProjectConfig(config),
|
||||
'klo',
|
||||
'klo@example.com',
|
||||
'seed test config',
|
||||
);
|
||||
}
|
||||
|
||||
it('rejects unsafe connection ids before loading a project', async () => {
|
||||
const io = makeIo();
|
||||
const loadProject = vi.fn(async () => {
|
||||
throw new Error('loadProject should not be called');
|
||||
});
|
||||
|
||||
await expect(
|
||||
runKloConnectionNotion(
|
||||
{
|
||||
command: 'pick',
|
||||
projectDir: '/tmp/project',
|
||||
connectionId: '../evil',
|
||||
mode: 'interactive',
|
||||
},
|
||||
io.io,
|
||||
{ loadProject },
|
||||
),
|
||||
).resolves.toBe(1);
|
||||
|
||||
expect(loadProject).not.toHaveBeenCalled();
|
||||
expect(io.stderr()).toContain('Unsafe connection id: ../evil');
|
||||
});
|
||||
|
||||
it('writes selected root_page_ids while preserving every other Notion connection field', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
const initialized = await initKloProject({ projectDir, projectName: 'warehouse' });
|
||||
await writeProjectConfig(projectDir, {
|
||||
...initialized.config,
|
||||
connections: {
|
||||
'notion-main': {
|
||||
driver: 'notion',
|
||||
auth_token_ref: 'env:NOTION_TOKEN',
|
||||
crawl_mode: 'all_accessible',
|
||||
root_page_ids: ['99999999-9999-9999-9999-999999999999'],
|
||||
root_database_ids: ['database-1'],
|
||||
root_data_source_ids: ['data-source-1'],
|
||||
max_pages_per_run: 12,
|
||||
max_knowledge_creates_per_run: 2,
|
||||
max_knowledge_updates_per_run: 7,
|
||||
last_successful_cursor: '{"phase":"all_accessible_pages","cursor":"cursor-1"}',
|
||||
unknown_future_field: 'keep-me',
|
||||
},
|
||||
},
|
||||
});
|
||||
const io = makeIo();
|
||||
|
||||
await expect(
|
||||
runKloConnectionNotion(
|
||||
{
|
||||
command: 'pick',
|
||||
projectDir,
|
||||
connectionId: 'notion-main',
|
||||
mode: 'non-interactive',
|
||||
rootPageIds: [
|
||||
'11111111-2222-3333-4444-555555555555',
|
||||
'66666666-7777-8888-9999-aaaaaaaaaaaa',
|
||||
],
|
||||
},
|
||||
io.io,
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
const yaml = await readFile(join(projectDir, 'klo.yaml'), 'utf-8');
|
||||
expect(yaml).toContain('crawl_mode: selected_roots');
|
||||
expect(yaml).toContain('root_page_ids:');
|
||||
expect(yaml).toContain('11111111-2222-3333-4444-555555555555');
|
||||
expect(yaml).toContain('66666666-7777-8888-9999-aaaaaaaaaaaa');
|
||||
expect(yaml).toContain('root_database_ids:');
|
||||
expect(yaml).toContain('database-1');
|
||||
expect(yaml).toContain('root_data_source_ids:');
|
||||
expect(yaml).toContain('data-source-1');
|
||||
expect(yaml).toContain('last_successful_cursor: \'{"phase":"all_accessible_pages","cursor":"cursor-1"}\'');
|
||||
expect(yaml).toContain('unknown_future_field: keep-me');
|
||||
expect(io.stdout()).toContain('Connection: notion-main');
|
||||
expect(io.stdout()).toContain('rootPageIds: 2');
|
||||
expect(io.stdout()).toContain('crawlMode: selected_roots');
|
||||
});
|
||||
|
||||
it('rejects empty writeback, missing connections, and non-Notion connections', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
const initialized = await initKloProject({ projectDir, projectName: 'warehouse' });
|
||||
await writeProjectConfig(projectDir, {
|
||||
...initialized.config,
|
||||
connections: {
|
||||
warehouse: {
|
||||
driver: 'postgres',
|
||||
url: 'env:DATABASE_URL',
|
||||
readonly: true,
|
||||
},
|
||||
},
|
||||
});
|
||||
const project = await loadKloProject({ projectDir });
|
||||
|
||||
await expect(applyNotionPickerWriteback(project, 'warehouse', [])).rejects.toThrow(
|
||||
'connection notion pick requires at least one root page id',
|
||||
);
|
||||
await expect(
|
||||
applyNotionPickerWriteback(project, 'missing', ['11111111-2222-3333-4444-555555555555']),
|
||||
).rejects.toThrow('Connection "missing" not found');
|
||||
await expect(
|
||||
applyNotionPickerWriteback(project, 'warehouse', ['11111111-2222-3333-4444-555555555555']),
|
||||
).rejects.toThrow('Connection "warehouse" is not a Notion connection');
|
||||
});
|
||||
|
||||
it('extracts picker page inputs from Notion search results', () => {
|
||||
expect(notionPickerPageFromSearchResult(notionPage(PAGE_IDS.architecture, 'Architecture', PAGE_IDS.engineering)))
|
||||
.toEqual({
|
||||
id: PAGE_IDS.architecture,
|
||||
title: 'Architecture',
|
||||
archived: false,
|
||||
parentId: PAGE_IDS.engineering,
|
||||
});
|
||||
|
||||
expect(
|
||||
notionPickerPageFromSearchResult({
|
||||
object: 'page',
|
||||
id: PAGE_IDS.engineering.replaceAll('-', ''),
|
||||
archived: true,
|
||||
parent: { type: 'workspace', workspace: true },
|
||||
properties: {},
|
||||
}),
|
||||
).toEqual({
|
||||
id: PAGE_IDS.engineering,
|
||||
title: 'Untitled',
|
||||
archived: true,
|
||||
parentId: null,
|
||||
});
|
||||
});
|
||||
|
||||
it('discovers visible pages up to the cap and reports cap state', async () => {
|
||||
const api = fakeNotionApi([
|
||||
notionPage(PAGE_IDS.engineering, 'Engineering'),
|
||||
notionPage(PAGE_IDS.architecture, 'Architecture', PAGE_IDS.engineering),
|
||||
notionPage('33333333-3333-3333-3333-333333333333', 'Onboarding', PAGE_IDS.engineering),
|
||||
]);
|
||||
|
||||
await expect(discoverNotionPickerPages(api, { cap: 2 })).resolves.toEqual({
|
||||
pages: [
|
||||
{ id: PAGE_IDS.engineering, title: 'Engineering', archived: false, parentId: null },
|
||||
{ id: PAGE_IDS.architecture, title: 'Architecture', archived: false, parentId: PAGE_IDS.engineering },
|
||||
],
|
||||
cappedAtCount: 2,
|
||||
warnings: [],
|
||||
});
|
||||
expect(api.search).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('keeps partial discovery results when Notion search fails after at least one page', async () => {
|
||||
const api: NotionPickerApi = {
|
||||
search: vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce({
|
||||
results: [notionPage(PAGE_IDS.engineering, 'Engineering')],
|
||||
hasMore: true,
|
||||
nextCursor: 'cursor-2',
|
||||
})
|
||||
.mockRejectedValueOnce(new Error('rate limit after first page')),
|
||||
retrieveBotUser: vi.fn(async () => ({ name: 'Notion bot' })),
|
||||
};
|
||||
|
||||
await expect(discoverNotionPickerPages(api)).resolves.toEqual({
|
||||
pages: [{ id: PAGE_IDS.engineering, title: 'Engineering', archived: false, parentId: null }],
|
||||
cappedAtCount: null,
|
||||
warnings: ['Notion search stopped early: rate limit after first page'],
|
||||
});
|
||||
});
|
||||
|
||||
it('uses the Notion workspace name when available and falls back to the connection id', async () => {
|
||||
await expect(resolveNotionWorkspaceLabel(fakeNotionApi([]), 'notion-main')).resolves.toBe('Design Workspace');
|
||||
await expect(
|
||||
resolveNotionWorkspaceLabel(
|
||||
{
|
||||
search: vi.fn(),
|
||||
retrieveBotUser: vi.fn(async () => {
|
||||
throw new Error('users.me unavailable');
|
||||
}),
|
||||
},
|
||||
'notion-main',
|
||||
),
|
||||
).resolves.toBe('notion-main');
|
||||
});
|
||||
|
||||
it('runs interactive discovery, warns about stale roots, renders the TUI, and saves selected roots', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
const initialized = await initKloProject({ projectDir, projectName: 'warehouse' });
|
||||
await writeProjectConfig(projectDir, {
|
||||
...initialized.config,
|
||||
connections: {
|
||||
'notion-main': {
|
||||
driver: 'notion',
|
||||
auth_token_ref: 'env:NOTION_TOKEN',
|
||||
crawl_mode: 'all_accessible',
|
||||
root_page_ids: [PAGE_IDS.stale],
|
||||
root_database_ids: ['database-1'],
|
||||
root_data_source_ids: ['data-source-1'],
|
||||
max_pages_per_run: 12,
|
||||
max_knowledge_creates_per_run: 2,
|
||||
max_knowledge_updates_per_run: 7,
|
||||
last_successful_cursor: null,
|
||||
},
|
||||
},
|
||||
});
|
||||
const api = fakeNotionApi([
|
||||
notionPage(PAGE_IDS.engineering, 'Engineering'),
|
||||
notionPage(PAGE_IDS.architecture, 'Architecture', PAGE_IDS.engineering),
|
||||
]);
|
||||
const renderPicker = vi.fn(async (input): Promise<PickerRenderResult> => {
|
||||
expect(input.connectionId).toBe('notion-main');
|
||||
expect(input.workspaceLabel).toBe('Design Workspace');
|
||||
expect(input.currentCrawlMode).toBe('all_accessible');
|
||||
expect(input.cappedAtCount).toBeNull();
|
||||
expect(input.initialState.preLoadWarnings).toEqual(['1 stored root_page_ids no longer visible']);
|
||||
return { kind: 'save', rootPageIds: [PAGE_IDS.engineering] };
|
||||
});
|
||||
const io = makeIo();
|
||||
|
||||
await expect(
|
||||
runKloConnectionNotion(
|
||||
{
|
||||
command: 'pick',
|
||||
projectDir,
|
||||
connectionId: 'notion-main',
|
||||
mode: 'interactive',
|
||||
},
|
||||
io.io,
|
||||
{
|
||||
env: { NOTION_TOKEN: 'ntn_test_token' },
|
||||
createNotionApi: vi.fn(() => api),
|
||||
renderPicker,
|
||||
},
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
const yaml = await readFile(join(projectDir, 'klo.yaml'), 'utf-8');
|
||||
expect(yaml).toContain('crawl_mode: selected_roots');
|
||||
expect(yaml).toContain(PAGE_IDS.engineering);
|
||||
expect(yaml).not.toContain(PAGE_IDS.stale);
|
||||
expect(io.stderr()).toContain('1 stored root_page_ids no longer visible');
|
||||
expect(io.stdout()).toContain('Connection: notion-main');
|
||||
expect(io.stdout()).toContain('rootPageIds: 1');
|
||||
});
|
||||
|
||||
it('passes partial-discovery warnings into the TUI banner state', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
const initialized = await initKloProject({ projectDir, projectName: 'warehouse' });
|
||||
await writeProjectConfig(projectDir, {
|
||||
...initialized.config,
|
||||
connections: {
|
||||
'notion-main': {
|
||||
driver: 'notion',
|
||||
auth_token_ref: 'env:NOTION_TOKEN',
|
||||
crawl_mode: 'selected_roots',
|
||||
root_page_ids: [PAGE_IDS.engineering],
|
||||
root_database_ids: [],
|
||||
root_data_source_ids: [],
|
||||
max_pages_per_run: 12,
|
||||
max_knowledge_creates_per_run: 2,
|
||||
max_knowledge_updates_per_run: 7,
|
||||
last_successful_cursor: null,
|
||||
},
|
||||
},
|
||||
});
|
||||
const api: NotionPickerApi = {
|
||||
search: vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce({
|
||||
results: [notionPage(PAGE_IDS.engineering, 'Engineering')],
|
||||
hasMore: true,
|
||||
nextCursor: 'cursor-2',
|
||||
})
|
||||
.mockRejectedValueOnce(new Error('rate limit after first page')),
|
||||
retrieveBotUser: vi.fn(async () => ({ name: 'Notion bot', bot: { workspace_name: 'Design Workspace' } })),
|
||||
};
|
||||
let renderInput: PickerRenderInput | undefined;
|
||||
const renderPicker = vi.fn(async (input: PickerRenderInput): Promise<PickerRenderResult> => {
|
||||
renderInput = input;
|
||||
return { kind: 'quit' };
|
||||
});
|
||||
const io = makeIo();
|
||||
|
||||
await expect(
|
||||
runKloConnectionNotion(
|
||||
{
|
||||
command: 'pick',
|
||||
projectDir,
|
||||
connectionId: 'notion-main',
|
||||
mode: 'interactive',
|
||||
},
|
||||
io.io,
|
||||
{
|
||||
env: { NOTION_TOKEN: 'ntn_test_token' },
|
||||
createNotionApi: vi.fn(() => api),
|
||||
renderPicker,
|
||||
},
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(renderPicker).toHaveBeenCalledOnce();
|
||||
if (!renderInput) {
|
||||
throw new Error('renderPicker was not called');
|
||||
}
|
||||
expect(renderInput.initialState.preLoadWarnings).toEqual(['Notion search stopped early: rate limit after first page']);
|
||||
expect(renderInput.initialState.tree.map((node) => node.title)).toEqual(['Engineering']);
|
||||
expect(io.stderr()).toContain('Notion search stopped early: rate limit after first page');
|
||||
expect(io.stdout()).toContain('No changes saved.');
|
||||
});
|
||||
|
||||
it('quits interactive mode without writing when the TUI returns quit', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
const initialized = await initKloProject({ projectDir, projectName: 'warehouse' });
|
||||
await writeProjectConfig(projectDir, {
|
||||
...initialized.config,
|
||||
connections: {
|
||||
'notion-main': {
|
||||
driver: 'notion',
|
||||
auth_token_ref: 'env:NOTION_TOKEN',
|
||||
crawl_mode: 'selected_roots',
|
||||
root_page_ids: [PAGE_IDS.engineering],
|
||||
root_database_ids: [],
|
||||
root_data_source_ids: [],
|
||||
max_pages_per_run: 12,
|
||||
max_knowledge_creates_per_run: 2,
|
||||
max_knowledge_updates_per_run: 7,
|
||||
last_successful_cursor: null,
|
||||
},
|
||||
},
|
||||
});
|
||||
const before = await readFile(join(projectDir, 'klo.yaml'), 'utf-8');
|
||||
const io = makeIo();
|
||||
|
||||
await expect(
|
||||
runKloConnectionNotion(
|
||||
{
|
||||
command: 'pick',
|
||||
projectDir,
|
||||
connectionId: 'notion-main',
|
||||
mode: 'interactive',
|
||||
},
|
||||
io.io,
|
||||
{
|
||||
env: { NOTION_TOKEN: 'ntn_test_token' },
|
||||
createNotionApi: vi.fn(() => fakeNotionApi([notionPage(PAGE_IDS.engineering, 'Engineering')])),
|
||||
renderPicker: vi.fn(async (): Promise<PickerRenderResult> => ({ kind: 'quit' })),
|
||||
},
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
await expect(readFile(join(projectDir, 'klo.yaml'), 'utf-8')).resolves.toBe(before);
|
||||
expect(io.stdout()).toContain('No changes saved.');
|
||||
});
|
||||
});
|
||||
278
packages/cli/src/commands/connection-notion.ts
Normal file
278
packages/cli/src/commands/connection-notion.ts
Normal file
|
|
@ -0,0 +1,278 @@
|
|||
import { parseNotionConnectionConfig, resolveNotionAuthToken } from '@klo/context/connections';
|
||||
import { type NotionApi, type NotionBotInfo, NotionClient } from '@klo/context/ingest';
|
||||
import {
|
||||
type KloLocalProject,
|
||||
type KloProjectConnectionConfig,
|
||||
loadKloProject,
|
||||
serializeKloProjectConfig,
|
||||
} from '@klo/context/project';
|
||||
import type { KloCliIo } from '../index.js';
|
||||
import { profileMark } from '../startup-profile.js';
|
||||
import { buildInitialState, buildPickerTree, type NotionPickerPageInput } from './connection-notion-tree.js';
|
||||
import {
|
||||
type NotionPickerTuiIo,
|
||||
type PickerRenderInput,
|
||||
type PickerRenderResult,
|
||||
renderNotionPickerTui,
|
||||
} from './connection-notion-tui.js';
|
||||
|
||||
profileMark('module:commands/connection-notion');
|
||||
|
||||
export type KloConnectionNotionArgs =
|
||||
| {
|
||||
command: 'pick';
|
||||
projectDir: string;
|
||||
connectionId: string;
|
||||
mode: 'interactive';
|
||||
}
|
||||
| {
|
||||
command: 'pick';
|
||||
projectDir: string;
|
||||
connectionId: string;
|
||||
mode: 'non-interactive';
|
||||
rootPageIds: string[];
|
||||
};
|
||||
|
||||
export type NotionPickerApi = Pick<NotionApi, 'search' | 'retrieveBotUser'>;
|
||||
export type { PickerRenderInput, PickerRenderResult };
|
||||
|
||||
interface KloConnectionNotionDeps {
|
||||
env?: Record<string, string | undefined>;
|
||||
loadProject?: typeof loadKloProject;
|
||||
createNotionApi?: (authToken: string) => NotionPickerApi;
|
||||
renderPicker?: (input: PickerRenderInput, io: NotionPickerTuiIo) => Promise<PickerRenderResult>;
|
||||
}
|
||||
|
||||
const NOTION_PICKER_PAGE_CAP = 5000;
|
||||
|
||||
function assertSafeConnectionId(connectionId: string): void {
|
||||
if (!/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(connectionId)) {
|
||||
throw new Error(`Unsafe connection id: ${connectionId}`);
|
||||
}
|
||||
}
|
||||
|
||||
export function normalizeNotionPageId(value: string): string {
|
||||
const trimmed = value.trim();
|
||||
const compact = trimmed.includes('-') ? trimmed.replace(/-/g, '') : trimmed;
|
||||
if (!/^[0-9a-fA-F]{32}$/.test(compact)) {
|
||||
throw new Error(`Invalid Notion page UUID: ${value}`);
|
||||
}
|
||||
const lower = compact.toLowerCase();
|
||||
return `${lower.slice(0, 8)}-${lower.slice(8, 12)}-${lower.slice(12, 16)}-${lower.slice(16, 20)}-${lower.slice(20)}`;
|
||||
}
|
||||
|
||||
function recordValue(value: unknown): Record<string, unknown> | null {
|
||||
return typeof value === 'object' && value !== null && !Array.isArray(value)
|
||||
? (value as Record<string, unknown>)
|
||||
: null;
|
||||
}
|
||||
|
||||
function extractTitleFromNotionPage(page: Record<string, unknown>): string {
|
||||
const properties = recordValue(page.properties);
|
||||
if (!properties) {
|
||||
return 'Untitled';
|
||||
}
|
||||
for (const property of Object.values(properties)) {
|
||||
const value = recordValue(property);
|
||||
if (!value || value.type !== 'title' || !Array.isArray(value.title)) {
|
||||
continue;
|
||||
}
|
||||
const text = value.title
|
||||
.map((part) => {
|
||||
const richText = recordValue(part);
|
||||
return typeof richText?.plain_text === 'string' ? richText.plain_text : '';
|
||||
})
|
||||
.join('')
|
||||
.trim();
|
||||
if (text.length > 0) {
|
||||
return text;
|
||||
}
|
||||
}
|
||||
return 'Untitled';
|
||||
}
|
||||
|
||||
function extractParentPageId(page: Record<string, unknown>): string | null {
|
||||
const parent = recordValue(page.parent);
|
||||
if (!parent || parent.type !== 'page_id' || typeof parent.page_id !== 'string') {
|
||||
return null;
|
||||
}
|
||||
return normalizeNotionPageId(parent.page_id);
|
||||
}
|
||||
|
||||
export function notionPickerPageFromSearchResult(result: Record<string, unknown>): NotionPickerPageInput {
|
||||
const id = typeof result.id === 'string' ? normalizeNotionPageId(result.id) : '';
|
||||
if (!id) {
|
||||
throw new Error('Notion page search result is missing id');
|
||||
}
|
||||
return {
|
||||
id,
|
||||
title: extractTitleFromNotionPage(result),
|
||||
archived: result.archived === true,
|
||||
parentId: extractParentPageId(result),
|
||||
};
|
||||
}
|
||||
|
||||
export async function discoverNotionPickerPages(
|
||||
api: NotionPickerApi,
|
||||
options: { cap?: number } = {},
|
||||
): Promise<{ pages: NotionPickerPageInput[]; cappedAtCount: number | null; warnings: string[] }> {
|
||||
const cap = options.cap ?? NOTION_PICKER_PAGE_CAP;
|
||||
const pages: NotionPickerPageInput[] = [];
|
||||
const warnings: string[] = [];
|
||||
let cursor: string | null | undefined = null;
|
||||
|
||||
while (pages.length < cap) {
|
||||
let response: Awaited<ReturnType<NotionPickerApi['search']>>;
|
||||
try {
|
||||
response = await api.search('page', cursor, Math.min(100, cap - pages.length));
|
||||
} catch (error) {
|
||||
if (pages.length === 0) {
|
||||
throw error;
|
||||
}
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
warnings.push(`Notion search stopped early: ${message}`);
|
||||
return { pages, cappedAtCount: null, warnings };
|
||||
}
|
||||
|
||||
for (const result of response.results) {
|
||||
pages.push(notionPickerPageFromSearchResult(result));
|
||||
if (pages.length >= cap) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!response.hasMore || !response.nextCursor || pages.length >= cap) {
|
||||
return {
|
||||
pages,
|
||||
cappedAtCount: response.hasMore ? cap : null,
|
||||
warnings,
|
||||
};
|
||||
}
|
||||
cursor = response.nextCursor;
|
||||
}
|
||||
|
||||
return { pages, cappedAtCount: cap, warnings };
|
||||
}
|
||||
|
||||
export async function resolveNotionWorkspaceLabel(api: NotionPickerApi, connectionId: string): Promise<string> {
|
||||
try {
|
||||
const bot = (await api.retrieveBotUser()) as NotionBotInfo;
|
||||
const workspaceName = typeof bot.bot?.workspace_name === 'string' ? bot.bot.workspace_name.trim() : '';
|
||||
if (workspaceName.length > 0) {
|
||||
return workspaceName;
|
||||
}
|
||||
const name = typeof bot.name === 'string' ? bot.name.trim() : '';
|
||||
return name.length > 0 ? name : connectionId;
|
||||
} catch {
|
||||
return connectionId;
|
||||
}
|
||||
}
|
||||
|
||||
function notionConnection(project: KloLocalProject, connectionId: string): KloProjectConnectionConfig {
|
||||
const connection = project.config.connections[connectionId];
|
||||
if (!connection) {
|
||||
throw new Error(`Connection "${connectionId}" not found`);
|
||||
}
|
||||
if (connection.driver !== 'notion') {
|
||||
throw new Error(`Connection "${connectionId}" is not a Notion connection`);
|
||||
}
|
||||
return connection;
|
||||
}
|
||||
|
||||
export async function applyNotionPickerWriteback(
|
||||
project: KloLocalProject,
|
||||
connectionId: string,
|
||||
rootPageIds: string[],
|
||||
): Promise<void> {
|
||||
if (rootPageIds.length === 0) {
|
||||
throw new Error('connection notion pick requires at least one root page id');
|
||||
}
|
||||
|
||||
const existing = notionConnection(project, connectionId);
|
||||
const nextConfig = {
|
||||
...project.config,
|
||||
connections: {
|
||||
...project.config.connections,
|
||||
[connectionId]: {
|
||||
...existing,
|
||||
crawl_mode: 'selected_roots',
|
||||
root_page_ids: rootPageIds,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
await project.fileStore.writeFile(
|
||||
'klo.yaml',
|
||||
serializeKloProjectConfig(nextConfig),
|
||||
'klo',
|
||||
'klo@example.com',
|
||||
`Pick Notion roots: ${connectionId} (${rootPageIds.length} pages)`,
|
||||
);
|
||||
}
|
||||
|
||||
export async function runKloConnectionNotion(
|
||||
args: KloConnectionNotionArgs,
|
||||
io: KloCliIo = process,
|
||||
deps: KloConnectionNotionDeps = {},
|
||||
): Promise<number> {
|
||||
try {
|
||||
assertSafeConnectionId(args.connectionId);
|
||||
const loadProject = deps.loadProject ?? loadKloProject;
|
||||
|
||||
if (args.mode === 'interactive') {
|
||||
const project = await loadProject({ projectDir: args.projectDir });
|
||||
const rawConnection = notionConnection(project, args.connectionId);
|
||||
const notion = parseNotionConnectionConfig(rawConnection);
|
||||
const authToken = await resolveNotionAuthToken(notion.auth_token_ref, { env: deps.env });
|
||||
const api = deps.createNotionApi ? deps.createNotionApi(authToken) : new NotionClient(authToken);
|
||||
const discovery = await discoverNotionPickerPages(api);
|
||||
const tree = buildPickerTree(discovery.pages);
|
||||
const initialState = buildInitialState({
|
||||
tree,
|
||||
existingRootPageIds: notion.root_page_ids,
|
||||
currentCrawlMode: notion.crawl_mode,
|
||||
});
|
||||
const preLoadWarnings = [...discovery.warnings, ...initialState.preLoadWarnings];
|
||||
const renderState =
|
||||
preLoadWarnings.length > 0
|
||||
? {
|
||||
...initialState,
|
||||
preLoadWarnings,
|
||||
}
|
||||
: initialState;
|
||||
for (const warning of preLoadWarnings) {
|
||||
io.stderr.write(`${warning}\n`);
|
||||
}
|
||||
const workspaceLabel = await resolveNotionWorkspaceLabel(api, args.connectionId);
|
||||
const result = await (deps.renderPicker ?? renderNotionPickerTui)(
|
||||
{
|
||||
initialState: renderState,
|
||||
connectionId: args.connectionId,
|
||||
workspaceLabel,
|
||||
cappedAtCount: discovery.cappedAtCount,
|
||||
currentCrawlMode: notion.crawl_mode,
|
||||
},
|
||||
io as NotionPickerTuiIo,
|
||||
);
|
||||
if (result.kind === 'quit') {
|
||||
io.stdout.write('No changes saved.\n');
|
||||
return 0;
|
||||
}
|
||||
await applyNotionPickerWriteback(project, args.connectionId, result.rootPageIds);
|
||||
io.stdout.write(`Connection: ${args.connectionId}\n`);
|
||||
io.stdout.write(`rootPageIds: ${result.rootPageIds.length}\n`);
|
||||
io.stdout.write('crawlMode: selected_roots\n');
|
||||
return 0;
|
||||
}
|
||||
|
||||
const project = await loadProject({ projectDir: args.projectDir });
|
||||
await applyNotionPickerWriteback(project, args.connectionId, args.rootPageIds);
|
||||
io.stdout.write(`Connection: ${args.connectionId}\n`);
|
||||
io.stdout.write(`rootPageIds: ${args.rootPageIds.length}\n`);
|
||||
io.stdout.write('crawlMode: selected_roots\n');
|
||||
return 0;
|
||||
} catch (error) {
|
||||
io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
26
packages/cli/src/commands/demo-commands.test.ts
Normal file
26
packages/cli/src/commands/demo-commands.test.ts
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import { resolveDemoCommandOptions } from './demo-commands.js';
|
||||
|
||||
describe('resolveDemoCommandOptions', () => {
|
||||
it('lets parent --no-input override a child default from optsWithGlobals', () => {
|
||||
const rootCommand = {
|
||||
opts: () => ({}),
|
||||
};
|
||||
const setupCommand = {
|
||||
parent: rootCommand,
|
||||
opts: () => ({ input: false }),
|
||||
getOptionValueSource: (name: string) => (name === 'input' ? 'cli' : undefined),
|
||||
};
|
||||
const demoCommand = {
|
||||
parent: setupCommand,
|
||||
opts: () => ({ input: true, mode: 'seeded' }),
|
||||
optsWithGlobals: () => ({ input: true, mode: 'seeded' }),
|
||||
getOptionValueSource: (name: string) => (name === 'input' ? 'default' : name === 'mode' ? 'default' : undefined),
|
||||
};
|
||||
|
||||
expect(resolveDemoCommandOptions<{ input: boolean; mode: string }>(demoCommand)).toEqual({
|
||||
input: false,
|
||||
mode: 'seeded',
|
||||
});
|
||||
});
|
||||
});
|
||||
273
packages/cli/src/commands/demo-commands.ts
Normal file
273
packages/cli/src/commands/demo-commands.ts
Normal file
|
|
@ -0,0 +1,273 @@
|
|||
import { type Command, Option } from '@commander-js/extra-typings';
|
||||
import {
|
||||
type CommandWithGlobalOptions,
|
||||
type KloCliCommandContext,
|
||||
resolveCommandProjectDirOverride,
|
||||
} from '../cli-program.js';
|
||||
import {
|
||||
type KloDemoArgs,
|
||||
type KloDemoInputMode,
|
||||
type KloDemoMode,
|
||||
type KloDemoOutputMode,
|
||||
} from '../demo.js';
|
||||
import { defaultDemoProjectDir } from '../demo-assets.js';
|
||||
import { resolveProjectDir } from '../project-dir.js';
|
||||
import { profileMark } from '../startup-profile.js';
|
||||
|
||||
profileMark('module:commands/demo-commands');
|
||||
|
||||
interface DemoOptions {
|
||||
plain?: boolean;
|
||||
json?: boolean;
|
||||
input?: boolean;
|
||||
projectDir?: string;
|
||||
}
|
||||
|
||||
function demoOutputMode(options: { plain?: boolean; json?: boolean }): KloDemoOutputMode {
|
||||
if (options.json === true) {
|
||||
return 'json';
|
||||
}
|
||||
if (options.plain === true) {
|
||||
return 'plain';
|
||||
}
|
||||
return 'viz';
|
||||
}
|
||||
|
||||
function demoDoctorOutputMode(options: { json?: boolean }): 'plain' | 'json' {
|
||||
return options.json === true ? 'json' : 'plain';
|
||||
}
|
||||
|
||||
function demoInspectOutputMode(options: { plain?: boolean; json?: boolean }): KloDemoOutputMode {
|
||||
if (options.json === true) {
|
||||
return 'json';
|
||||
}
|
||||
return 'plain';
|
||||
}
|
||||
|
||||
function demoInputMode(options: { input?: boolean }): { inputMode?: KloDemoInputMode } {
|
||||
return options.input === false ? { inputMode: 'disabled' } : {};
|
||||
}
|
||||
|
||||
function demoProjectDir(options: { projectDir?: string }, command: CommandWithGlobalOptions): string {
|
||||
return resolveProjectDir(
|
||||
options.projectDir ?? resolveCommandProjectDirOverride(command),
|
||||
defaultDemoProjectDir(),
|
||||
);
|
||||
}
|
||||
|
||||
type CommandOptionSourceReader = {
|
||||
getOptionValueSource?: (name: string) => string | undefined;
|
||||
parent?: unknown;
|
||||
};
|
||||
|
||||
function inheritedOptionSource(command: CommandOptionSourceReader, key: string): string | undefined {
|
||||
let current = command.parent as (CommandOptionSourceReader & { opts?: () => Record<string, unknown> }) | undefined;
|
||||
while (current) {
|
||||
const source = current.getOptionValueSource?.(key);
|
||||
if (source !== undefined) {
|
||||
return source;
|
||||
}
|
||||
current = current.parent as (CommandOptionSourceReader & { opts?: () => Record<string, unknown> }) | undefined;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function definedOptions(
|
||||
options: Record<string, unknown>,
|
||||
inherited: Record<string, unknown> = {},
|
||||
command?: CommandOptionSourceReader,
|
||||
): Record<string, unknown> {
|
||||
return Object.fromEntries(
|
||||
Object.entries(options).filter(([key, value]) => {
|
||||
if (value === undefined) return false;
|
||||
if (key === 'input' && value === true && inherited.input === false) return false;
|
||||
if (
|
||||
key === 'mode' &&
|
||||
command?.getOptionValueSource?.(key) === 'default' &&
|
||||
inherited[key] !== undefined &&
|
||||
inherited[key] !== value &&
|
||||
inheritedOptionSource(command, key) === 'cli'
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
export function resolveDemoCommandOptions<T>(command: { opts: () => T; optsWithGlobals?: () => T; parent?: unknown }): T {
|
||||
const chain: Array<{ opts?: () => Record<string, unknown>; parent?: unknown }> = [];
|
||||
let current = command.parent as { opts?: () => Record<string, unknown>; parent?: unknown } | undefined;
|
||||
while (current) {
|
||||
chain.unshift(current);
|
||||
current = current.parent as { opts?: () => Record<string, unknown>; parent?: unknown } | undefined;
|
||||
}
|
||||
const inherited = Object.assign({}, ...chain.map((parent) => definedOptions(parent.opts?.() ?? {})));
|
||||
|
||||
if (command.optsWithGlobals) {
|
||||
const withGlobals = {
|
||||
...inherited,
|
||||
...definedOptions(command.optsWithGlobals() as Record<string, unknown>, inherited, command),
|
||||
};
|
||||
return {
|
||||
...withGlobals,
|
||||
...definedOptions(command.opts() as Record<string, unknown>, withGlobals, command),
|
||||
} as T;
|
||||
}
|
||||
|
||||
return { ...inherited, ...definedOptions(command.opts() as Record<string, unknown>, inherited, command) } as T;
|
||||
}
|
||||
|
||||
async function runDemoArgs(context: KloCliCommandContext, args: KloDemoArgs): Promise<void> {
|
||||
const runner = context.deps.demo ?? (await import('../demo.js')).runKloDemo;
|
||||
context.setExitCode(await runner(args, context.io));
|
||||
}
|
||||
|
||||
export function registerDemoCommands(
|
||||
program: Command,
|
||||
context: KloCliCommandContext,
|
||||
options: { description?: string } = {},
|
||||
): void {
|
||||
const demo = program
|
||||
.command('demo')
|
||||
.description(options.description ?? 'Run the pre-seeded KLO demo or a full LLM-backed demo')
|
||||
.addOption(
|
||||
new Option('--mode <mode>', 'Demo mode: seeded (default), replay, or full')
|
||||
.choices(['seeded', 'replay', 'full'])
|
||||
.default('seeded'),
|
||||
)
|
||||
.option('--project-dir <path>', 'Demo project directory')
|
||||
.addOption(new Option('--plain', 'Print plain text output instead of the visual demo').conflicts('json'))
|
||||
.addOption(new Option('--json', 'Print JSON output').conflicts('plain'))
|
||||
.option('--no-input', 'Disable interactive terminal input')
|
||||
.showHelpAfterError()
|
||||
.action(async (options: { mode: 'seeded' | 'replay' | 'full' } & DemoOptions, command) => {
|
||||
const resolvedOptions = resolveDemoCommandOptions<typeof options>(command);
|
||||
await runDemoArgs(context, {
|
||||
command: resolvedOptions.mode,
|
||||
projectDir: demoProjectDir(resolvedOptions, command),
|
||||
outputMode: demoOutputMode(resolvedOptions),
|
||||
...demoInputMode(resolvedOptions),
|
||||
});
|
||||
});
|
||||
|
||||
demo
|
||||
.command('init')
|
||||
.description('Initialize the packaged demo project')
|
||||
.option('--project-dir <path>', 'Demo project directory')
|
||||
.option('--force', 'Recreate an existing demo project', false)
|
||||
.option('--no-input', 'Disable interactive terminal input')
|
||||
.action(async (_options, command: { opts: () => { projectDir?: string; force?: boolean; input?: boolean } }) => {
|
||||
const options = resolveDemoCommandOptions(command);
|
||||
await runDemoArgs(context, {
|
||||
command: 'init',
|
||||
projectDir: demoProjectDir(options, command),
|
||||
force: options.force === true,
|
||||
...demoInputMode(options),
|
||||
});
|
||||
});
|
||||
|
||||
demo
|
||||
.command('reset')
|
||||
.description('Reset the packaged demo project')
|
||||
.option('--project-dir <path>', 'Demo project directory')
|
||||
.option('--force', 'Recreate the demo project without prompting', false)
|
||||
.option('--no-input', 'Disable interactive terminal input')
|
||||
.action(async (_options, command: { opts: () => { projectDir?: string; force?: boolean; input?: boolean } }) => {
|
||||
const options = resolveDemoCommandOptions(command);
|
||||
await runDemoArgs(context, {
|
||||
command: 'reset',
|
||||
projectDir: demoProjectDir(options, command),
|
||||
force: options.force === true,
|
||||
...demoInputMode(options),
|
||||
});
|
||||
});
|
||||
|
||||
demo
|
||||
.command('replay')
|
||||
.description('Replay the packaged demo memory-flow')
|
||||
.option('--project-dir <path>', 'Demo project directory')
|
||||
.addOption(new Option('--plain', 'Print plain text output instead of the visual demo').conflicts('json'))
|
||||
.addOption(new Option('--json', 'Print JSON output').conflicts('plain'))
|
||||
.option('--no-input', 'Disable interactive terminal input')
|
||||
.action(async (_options, command: { opts: () => DemoOptions }) => {
|
||||
const options = resolveDemoCommandOptions(command);
|
||||
await runDemoArgs(context, {
|
||||
command: 'replay',
|
||||
projectDir: demoProjectDir(options, command),
|
||||
outputMode: demoOutputMode(options),
|
||||
...demoInputMode(options),
|
||||
});
|
||||
});
|
||||
|
||||
demo
|
||||
.command('scan')
|
||||
.description('Run the packaged demo scan')
|
||||
.option('--project-dir <path>', 'Demo project directory')
|
||||
.option('--no-input', 'Disable interactive terminal input')
|
||||
.action(async (_options, command: { opts: () => { projectDir?: string; input?: boolean } }) => {
|
||||
const options = resolveDemoCommandOptions(command);
|
||||
await runDemoArgs(context, {
|
||||
command: 'scan',
|
||||
projectDir: demoProjectDir(options, command),
|
||||
...demoInputMode(options),
|
||||
});
|
||||
});
|
||||
|
||||
demo
|
||||
.command('inspect')
|
||||
.description('Inspect packaged demo outputs')
|
||||
.option('--project-dir <path>', 'Demo project directory')
|
||||
.addOption(new Option('--plain', 'Print plain text output').conflicts('json'))
|
||||
.addOption(new Option('--json', 'Print JSON output').conflicts('plain'))
|
||||
.option('--no-input', 'Disable interactive terminal input')
|
||||
.action(async (_options, command: { opts: () => DemoOptions }) => {
|
||||
const options = resolveDemoCommandOptions(command);
|
||||
await runDemoArgs(context, {
|
||||
command: 'inspect',
|
||||
projectDir: demoProjectDir(options, command),
|
||||
outputMode: demoInspectOutputMode(options),
|
||||
...demoInputMode(options),
|
||||
});
|
||||
});
|
||||
|
||||
demo
|
||||
.command('doctor')
|
||||
.description('Check packaged demo readiness')
|
||||
.option('--project-dir <path>', 'Demo project directory')
|
||||
.addOption(new Option('--plain', 'Print plain text output').conflicts('json'))
|
||||
.addOption(new Option('--json', 'Print JSON output').conflicts('plain'))
|
||||
.option('--no-input', 'Disable interactive terminal input')
|
||||
.action(async (_options, command: { opts: () => DemoOptions }) => {
|
||||
const options = resolveDemoCommandOptions(command);
|
||||
await runDemoArgs(context, {
|
||||
command: 'doctor',
|
||||
projectDir: demoProjectDir(options, command),
|
||||
outputMode: demoDoctorOutputMode(options),
|
||||
...demoInputMode(options),
|
||||
});
|
||||
});
|
||||
|
||||
demo
|
||||
.command('ingest')
|
||||
.description('Run packaged demo ingest')
|
||||
.addOption(
|
||||
new Option('--mode <mode>', 'Demo ingest mode: full or seeded')
|
||||
.choices(['full', 'seeded'])
|
||||
.default('full'),
|
||||
)
|
||||
.option('--project-dir <path>', 'Demo project directory')
|
||||
.addOption(new Option('--plain', 'Print plain text output instead of the visual demo').conflicts('json'))
|
||||
.addOption(new Option('--json', 'Print JSON output').conflicts('plain'))
|
||||
.option('--no-input', 'Disable interactive terminal input')
|
||||
.action(async (_options, command: { opts: () => { mode: KloDemoMode } & DemoOptions }) => {
|
||||
const options = resolveDemoCommandOptions(command);
|
||||
await runDemoArgs(context, {
|
||||
command: 'ingest',
|
||||
mode: options.mode,
|
||||
projectDir: demoProjectDir(options, command),
|
||||
outputMode: demoOutputMode(options),
|
||||
...demoInputMode(options),
|
||||
});
|
||||
});
|
||||
}
|
||||
53
packages/cli/src/commands/doctor-commands.ts
Normal file
53
packages/cli/src/commands/doctor-commands.ts
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
import type { Command } from '@commander-js/extra-typings';
|
||||
import { type CommandWithGlobalOptions, type KloCliCommandContext, resolveCommandProjectDir } from '../cli-program.js';
|
||||
import type { KloDoctorArgs } from '../doctor.js';
|
||||
import { profileMark } from '../startup-profile.js';
|
||||
|
||||
profileMark('module:commands/doctor-commands');
|
||||
|
||||
function outputMode(options: { json?: boolean }): 'plain' | 'json' {
|
||||
return options.json === true ? 'json' : 'plain';
|
||||
}
|
||||
|
||||
function inputMode(options: { input?: boolean }): { inputMode?: 'disabled' } {
|
||||
return options.input === false ? { inputMode: 'disabled' } : {};
|
||||
}
|
||||
|
||||
async function runDoctorArgs(context: KloCliCommandContext, args: KloDoctorArgs): Promise<void> {
|
||||
const runner = context.deps.doctor ?? (await import('../doctor.js')).runKloDoctor;
|
||||
context.setExitCode(await runner(args, context.io));
|
||||
}
|
||||
|
||||
export function registerDoctorCommands(program: Command, context: KloCliCommandContext): void {
|
||||
const doctor = program
|
||||
.command('doctor')
|
||||
.description('Check KLO setup, project, and demo readiness')
|
||||
.option('--json', 'Print JSON output', false)
|
||||
.option('--no-input', 'Disable interactive terminal input')
|
||||
.action(async (options: { json?: boolean; input?: boolean }, command) => {
|
||||
await runDoctorArgs(context, {
|
||||
command: 'project',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
outputMode: outputMode(options),
|
||||
...inputMode(options),
|
||||
});
|
||||
});
|
||||
|
||||
doctor
|
||||
.command('setup')
|
||||
.description('Check KLO install, build, and local runtime readiness')
|
||||
.option('--json', 'Print JSON output', false)
|
||||
.option('--no-input', 'Disable interactive terminal input')
|
||||
.action(
|
||||
async (
|
||||
_options: { json?: boolean; input?: boolean },
|
||||
command: CommandWithGlobalOptions,
|
||||
) => {
|
||||
const options = (command.optsWithGlobals ? command.optsWithGlobals() : command.opts()) as {
|
||||
json?: boolean;
|
||||
input?: boolean;
|
||||
};
|
||||
await runDoctorArgs(context, { command: 'setup', outputMode: outputMode(options), ...inputMode(options) });
|
||||
},
|
||||
);
|
||||
}
|
||||
171
packages/cli/src/commands/ingest-commands.ts
Normal file
171
packages/cli/src/commands/ingest-commands.ts
Normal file
|
|
@ -0,0 +1,171 @@
|
|||
import { resolve } from 'node:path';
|
||||
import { type Command, Option } from '@commander-js/extra-typings';
|
||||
import { type KloCliCommandContext, type OutputModeOptions, resolveCommandProjectDir } from '../cli-program.js';
|
||||
import type { KloCliDeps, KloCliIo } from '../index.js';
|
||||
import type { KloIngestArgs, KloIngestOutputMode } from '../ingest.js';
|
||||
import { profileMark } from '../startup-profile.js';
|
||||
|
||||
profileMark('module:commands/ingest-commands');
|
||||
|
||||
interface IngestCommandOptions {
|
||||
runIngestWithProgress: (
|
||||
args: KloIngestArgs,
|
||||
io: KloCliIo,
|
||||
deps: KloCliDeps,
|
||||
defaultRunIngest: (args: KloIngestArgs, io: KloCliIo) => Promise<number>,
|
||||
) => Promise<number>;
|
||||
}
|
||||
|
||||
function outputMode(options: OutputModeOptions): KloIngestOutputMode {
|
||||
if (options.json === true) {
|
||||
return 'json';
|
||||
}
|
||||
if (options.viz === true) {
|
||||
return 'viz';
|
||||
}
|
||||
return 'plain';
|
||||
}
|
||||
|
||||
function watchOutputMode(options: OutputModeOptions): KloIngestOutputMode {
|
||||
if (options.json === true) {
|
||||
return 'json';
|
||||
}
|
||||
if (options.plain === true) {
|
||||
return 'plain';
|
||||
}
|
||||
return 'viz';
|
||||
}
|
||||
|
||||
function inputMode(options: OutputModeOptions): Pick<KloIngestArgs, 'inputMode'> {
|
||||
return options.input === false ? { inputMode: 'disabled' } : {};
|
||||
}
|
||||
|
||||
async function runIngestArgs(
|
||||
context: KloCliCommandContext,
|
||||
args: KloIngestArgs,
|
||||
options: IngestCommandOptions,
|
||||
): Promise<void> {
|
||||
const { runKloIngest } = await import('../ingest.js');
|
||||
context.setExitCode(await options.runIngestWithProgress(args, context.io, context.deps, runKloIngest));
|
||||
}
|
||||
|
||||
export function registerIngestCommands(
|
||||
program: Command,
|
||||
context: KloCliCommandContext,
|
||||
commandOptions: IngestCommandOptions,
|
||||
): void {
|
||||
const ingest = program
|
||||
.command('ingest')
|
||||
.description('Run or inspect local ingest memory-flow output')
|
||||
.showHelpAfterError();
|
||||
|
||||
ingest.hook('preAction', (_thisCommand, actionCommand) => {
|
||||
context.writeDebug?.('ingest', actionCommand);
|
||||
});
|
||||
|
||||
ingest
|
||||
.command('run')
|
||||
.description('Run local ingest for one configured connection and source adapter')
|
||||
.requiredOption('--connection-id <connectionId>', 'KLO connection id')
|
||||
.requiredOption('--adapter <adapter>', 'Ingest source adapter name')
|
||||
.option('--source-dir <path>', 'Directory containing source files')
|
||||
.option('--database-introspection-url <url>', 'Daemon URL for live-database introspection')
|
||||
.option('--debug-llm-request-file <path>', 'Write sanitized LLM request structure to a JSONL file')
|
||||
.option('--report-file <path>', 'Unsupported for ingest run; use ingest status/watch instead')
|
||||
.addOption(new Option('--plain', 'Print plain text output').conflicts(['json', 'viz']))
|
||||
.addOption(new Option('--json', 'Print JSON output').conflicts(['plain', 'viz']))
|
||||
.addOption(new Option('--viz', 'Render memory-flow TUI output').conflicts(['plain', 'json']))
|
||||
.option('--no-input', 'Disable interactive terminal input for visualization')
|
||||
.action(async (options, command) => {
|
||||
if (options.reportFile) {
|
||||
throw new Error('--report-file is only supported for ingest status/watch');
|
||||
}
|
||||
await runIngestArgs(
|
||||
context,
|
||||
{
|
||||
command: 'run',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
connectionId: options.connectionId,
|
||||
adapter: options.adapter,
|
||||
sourceDir: options.sourceDir ? resolve(options.sourceDir) : undefined,
|
||||
databaseIntrospectionUrl: options.databaseIntrospectionUrl || undefined,
|
||||
...(options.debugLlmRequestFile ? { debugLlmRequestFile: resolve(options.debugLlmRequestFile) } : {}),
|
||||
outputMode: outputMode(options),
|
||||
...inputMode(options),
|
||||
},
|
||||
commandOptions,
|
||||
);
|
||||
});
|
||||
|
||||
ingest
|
||||
.command('status')
|
||||
.description('Print status for the latest or selected stored local ingest run or report file')
|
||||
.argument('[runId]', 'Local ingest run id, report id, run id, or job id')
|
||||
.option('--report-file <path>', 'Bundle ingest report JSON file to render')
|
||||
.addOption(new Option('--plain', 'Print plain text output').conflicts(['json', 'viz']))
|
||||
.addOption(new Option('--json', 'Print JSON output').conflicts(['plain', 'viz']))
|
||||
.addOption(new Option('--viz', 'Render memory-flow TUI output').conflicts(['plain', 'json']))
|
||||
.option('--no-input', 'Disable interactive terminal input for visualization')
|
||||
.action(async (runId: string | undefined, options, command) => {
|
||||
await runIngestArgs(
|
||||
context,
|
||||
{
|
||||
command: 'status',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
...(runId ? { runId } : {}),
|
||||
...(options.reportFile ? { reportFile: resolve(options.reportFile) } : {}),
|
||||
outputMode: outputMode(options),
|
||||
...inputMode(options),
|
||||
},
|
||||
commandOptions,
|
||||
);
|
||||
});
|
||||
|
||||
ingest
|
||||
.command('watch')
|
||||
.description('Open the latest or selected stored ingest visual report')
|
||||
.argument('[runId]', 'Local ingest run id, report id, run id, or job id')
|
||||
.option('--report-file <path>', 'Bundle ingest report JSON file to render')
|
||||
.addOption(new Option('--plain', 'Print plain text output').conflicts(['json', 'viz']))
|
||||
.addOption(new Option('--json', 'Print JSON output').conflicts(['plain', 'viz']))
|
||||
.addOption(new Option('--viz', 'Render memory-flow TUI output').conflicts(['plain', 'json']))
|
||||
.option('--no-input', 'Disable interactive terminal input for visualization')
|
||||
.action(async (runId: string | undefined, options, command) => {
|
||||
await runIngestArgs(
|
||||
context,
|
||||
{
|
||||
command: 'watch',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
...(runId ? { runId } : {}),
|
||||
...(options.reportFile ? { reportFile: resolve(options.reportFile) } : {}),
|
||||
outputMode: watchOutputMode(options),
|
||||
...inputMode(options),
|
||||
},
|
||||
commandOptions,
|
||||
);
|
||||
});
|
||||
|
||||
ingest
|
||||
.command('replay')
|
||||
.description('Replay a stored ingest run or bundle report through memory-flow output')
|
||||
.argument('<runId>', 'Local ingest run id, report id, run id, or job id')
|
||||
.option('--report-file <path>', 'Bundle ingest report JSON file to render')
|
||||
.addOption(new Option('--plain', 'Print plain text output').conflicts(['json', 'viz']))
|
||||
.addOption(new Option('--json', 'Print JSON output').conflicts(['plain', 'viz']))
|
||||
.addOption(new Option('--viz', 'Render memory-flow TUI output').conflicts(['plain', 'json']))
|
||||
.option('--no-input', 'Disable interactive terminal input for visualization')
|
||||
.action(async (runId: string, options, command) => {
|
||||
await runIngestArgs(
|
||||
context,
|
||||
{
|
||||
command: 'replay',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
runId,
|
||||
...(options.reportFile ? { reportFile: resolve(options.reportFile) } : {}),
|
||||
outputMode: outputMode(options),
|
||||
...inputMode(options),
|
||||
},
|
||||
commandOptions,
|
||||
);
|
||||
});
|
||||
}
|
||||
90
packages/cli/src/commands/knowledge-commands.ts
Normal file
90
packages/cli/src/commands/knowledge-commands.ts
Normal file
|
|
@ -0,0 +1,90 @@
|
|||
import { type Command, Option } from '@commander-js/extra-typings';
|
||||
import { collectOption, type KloCliCommandContext, resolveCommandProjectDir } from '../cli-program.js';
|
||||
import { wikiWriteCommandSchema } from '../command-schemas.js';
|
||||
import type { KloKnowledgeArgs } from '../knowledge.js';
|
||||
import { profileMark } from '../startup-profile.js';
|
||||
|
||||
profileMark('module:commands/knowledge-commands');
|
||||
|
||||
async function runKnowledgeArgs(context: KloCliCommandContext, args: KloKnowledgeArgs): Promise<void> {
|
||||
const runner = context.deps.knowledge ?? (await import('../knowledge.js')).runKloKnowledge;
|
||||
context.setExitCode(await runner(args, context.io));
|
||||
}
|
||||
|
||||
export function registerWikiCommands(program: Command, context: KloCliCommandContext): void {
|
||||
const wiki = program
|
||||
.command('wiki')
|
||||
.description('List, read, search, or write local wiki pages')
|
||||
.showHelpAfterError()
|
||||
.addHelpText(
|
||||
'after',
|
||||
'\nProject directory defaults to KLO_PROJECT_DIR when set, otherwise the current working directory.\n',
|
||||
);
|
||||
|
||||
wiki
|
||||
.command('list')
|
||||
.description('List local wiki pages')
|
||||
.option('--user-id <id>', 'Local user id', 'local')
|
||||
.action(async (options: { userId: string }, command) => {
|
||||
await runKnowledgeArgs(context, {
|
||||
command: 'list',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
userId: options.userId,
|
||||
});
|
||||
});
|
||||
|
||||
wiki
|
||||
.command('read')
|
||||
.description('Read one local wiki page')
|
||||
.argument('<key>', 'Wiki page key')
|
||||
.option('--user-id <id>', 'Local user id', 'local')
|
||||
.action(async (key: string, options: { userId: string }, command) => {
|
||||
await runKnowledgeArgs(context, {
|
||||
command: 'read',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
key,
|
||||
userId: options.userId,
|
||||
});
|
||||
});
|
||||
|
||||
wiki
|
||||
.command('search')
|
||||
.description('Search local wiki pages')
|
||||
.argument('<query>', 'Search query')
|
||||
.option('--user-id <id>', 'Local user id', 'local')
|
||||
.action(async (query: string, options: { userId: string }, command) => {
|
||||
await runKnowledgeArgs(context, {
|
||||
command: 'search',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
query,
|
||||
userId: options.userId,
|
||||
});
|
||||
});
|
||||
|
||||
wiki
|
||||
.command('write')
|
||||
.description('Write one local wiki page')
|
||||
.argument('<key>', 'Wiki page key')
|
||||
.option('--user-id <id>', 'Local user id', 'local')
|
||||
.addOption(new Option('--scope <scope>', 'global or user').choices(['global', 'user']).default('global'))
|
||||
.requiredOption('--summary <summary>', 'Wiki summary')
|
||||
.requiredOption('--content <content>', 'Wiki content')
|
||||
.option('--tag <tag>', 'Wiki tag; repeatable', collectOption, [])
|
||||
.option('--ref <ref>', 'Wiki ref; repeatable', collectOption, [])
|
||||
.option('--sl-ref <ref>', 'Semantic-layer ref; repeatable', collectOption, [])
|
||||
.action(async (key: string, options, command) => {
|
||||
const args = wikiWriteCommandSchema.parse({
|
||||
command: 'write',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
key,
|
||||
scope: options.scope === 'user' ? 'USER' : 'GLOBAL',
|
||||
userId: options.userId,
|
||||
summary: options.summary,
|
||||
content: options.content,
|
||||
tags: options.tag,
|
||||
refs: options.ref,
|
||||
slRefs: options.slRef,
|
||||
});
|
||||
await runKnowledgeArgs(context, args);
|
||||
});
|
||||
}
|
||||
109
packages/cli/src/commands/public-ingest-commands.ts
Normal file
109
packages/cli/src/commands/public-ingest-commands.ts
Normal file
|
|
@ -0,0 +1,109 @@
|
|||
import { InvalidArgumentError, type Command } from '@commander-js/extra-typings';
|
||||
import { type KloCliCommandContext, resolveCommandProjectDir } from '../cli-program.js';
|
||||
import { publicIngestReadCommandSchema, publicIngestRunCommandSchema } from '../command-schemas.js';
|
||||
import type { KloPublicIngestArgs, KloPublicIngestInputMode } from '../public-ingest.js';
|
||||
import { profileMark } from '../startup-profile.js';
|
||||
|
||||
profileMark('module:commands/public-ingest-commands');
|
||||
|
||||
interface PublicIngestOptions {
|
||||
all?: boolean;
|
||||
json?: boolean;
|
||||
input?: boolean;
|
||||
}
|
||||
|
||||
function inputMode(options: { input?: boolean }): KloPublicIngestInputMode {
|
||||
return options.input === false ? 'disabled' : 'auto';
|
||||
}
|
||||
|
||||
async function runPublicIngestArgs(context: KloCliCommandContext, args: KloPublicIngestArgs): Promise<void> {
|
||||
const runner = context.deps.publicIngest ?? (await import('../public-ingest.js')).runKloPublicIngest;
|
||||
context.setExitCode(await runner(args, context.io));
|
||||
}
|
||||
|
||||
function parsePublicIngestConnectionId(value: string): string {
|
||||
if (value === 'run') {
|
||||
throw new InvalidArgumentError('run is reserved; use klo dev ingest run for low-level adapter syntax');
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
export function registerPublicIngestCommands(program: Command, context: KloCliCommandContext): void {
|
||||
const ingest = program
|
||||
.command('ingest')
|
||||
.description('Build and refresh KLO context from configured sources')
|
||||
.usage('[options] [connectionId]')
|
||||
.argument('[connectionId]', 'Connection id to ingest', parsePublicIngestConnectionId)
|
||||
.option('--all', 'Ingest every eligible configured source', false)
|
||||
.option('--json', 'Print JSON output', false)
|
||||
.option('--no-input', 'Disable interactive terminal input')
|
||||
.addHelpText(
|
||||
'after',
|
||||
[
|
||||
'',
|
||||
'Examples:',
|
||||
' klo ingest <connectionId> [options]',
|
||||
' klo ingest --all [options]',
|
||||
' klo ingest status [runId] [options]',
|
||||
' klo ingest watch [runId] [options]',
|
||||
'',
|
||||
'Project directory defaults to KLO_PROJECT_DIR when set, otherwise the current working directory.',
|
||||
'',
|
||||
].join('\n'),
|
||||
)
|
||||
.showHelpAfterError()
|
||||
.hook('preAction', (_thisCommand, actionCommand) => {
|
||||
context.writeDebug?.('ingest', actionCommand);
|
||||
})
|
||||
.action(async (connectionId: string | undefined, _options: PublicIngestOptions, command) => {
|
||||
const options = command.opts();
|
||||
if (options.all === true && connectionId) {
|
||||
throw new Error('klo ingest accepts either --all or <connectionId>, not both');
|
||||
}
|
||||
const args = publicIngestRunCommandSchema.parse({
|
||||
command: 'run',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
...(connectionId ? { targetConnectionId: connectionId } : {}),
|
||||
all: options.all === true,
|
||||
json: options.json === true,
|
||||
inputMode: inputMode(options),
|
||||
});
|
||||
await runPublicIngestArgs(context, args);
|
||||
});
|
||||
|
||||
ingest
|
||||
.command('status')
|
||||
.description('Print status for the latest or selected public ingest run')
|
||||
.argument('[runId]', 'Public ingest run id')
|
||||
.option('--json', 'Print JSON output', false)
|
||||
.option('--no-input', 'Disable interactive terminal input')
|
||||
.action(async (runId: string | undefined, _options: PublicIngestOptions, command) => {
|
||||
const options = (command.optsWithGlobals ? command.optsWithGlobals() : command.opts()) as PublicIngestOptions;
|
||||
const args = publicIngestReadCommandSchema.parse({
|
||||
command: 'status',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
...(runId ? { runId } : {}),
|
||||
json: options.json === true,
|
||||
inputMode: inputMode(options),
|
||||
});
|
||||
await runPublicIngestArgs(context, args);
|
||||
});
|
||||
|
||||
ingest
|
||||
.command('watch')
|
||||
.description('Open the latest or selected public ingest visual report')
|
||||
.argument('[runId]', 'Public ingest run id')
|
||||
.option('--json', 'Print JSON output instead of the visual report', false)
|
||||
.option('--no-input', 'Disable interactive terminal input')
|
||||
.action(async (runId: string | undefined, _options: PublicIngestOptions, command) => {
|
||||
const options = (command.optsWithGlobals ? command.optsWithGlobals() : command.opts()) as PublicIngestOptions;
|
||||
const args = publicIngestReadCommandSchema.parse({
|
||||
command: 'watch',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
...(runId ? { runId } : {}),
|
||||
json: options.json === true,
|
||||
inputMode: inputMode(options),
|
||||
});
|
||||
await runPublicIngestArgs(context, args);
|
||||
});
|
||||
}
|
||||
353
packages/cli/src/commands/scan-commands.ts
Normal file
353
packages/cli/src/commands/scan-commands.ts
Normal file
|
|
@ -0,0 +1,353 @@
|
|||
import { type Command, InvalidArgumentError, Option } from '@commander-js/extra-typings';
|
||||
import { type KloCliCommandContext, parsePositiveIntegerOption, resolveCommandProjectDir } from '../cli-program.js';
|
||||
import type { KloScanArgs } from '../scan.js';
|
||||
import { profileMark } from '../startup-profile.js';
|
||||
|
||||
profileMark('module:commands/scan-commands');
|
||||
|
||||
async function runScanArgs(context: KloCliCommandContext, args: KloScanArgs): Promise<void> {
|
||||
const runner = context.deps.scan ?? (await import('../scan.js')).runKloScan;
|
||||
context.setExitCode(await runner(args, context.io));
|
||||
}
|
||||
|
||||
type KloScanModeOption = Extract<KloScanArgs, { command: 'run' }>['mode'];
|
||||
|
||||
function parseScanModeOption(value: string): KloScanModeOption {
|
||||
if (value === 'structural' || value === 'enriched' || value === 'relationships') {
|
||||
return value;
|
||||
}
|
||||
throw new InvalidArgumentError('Allowed choices are structural, enriched, relationships');
|
||||
}
|
||||
|
||||
type KloRelationshipStatusOption = Extract<KloScanArgs, { command: 'relationships' }>['status'];
|
||||
type KloRelationshipFeedbackDecisionOption = Extract<KloScanArgs, { command: 'relationshipFeedback' }>['decision'];
|
||||
|
||||
function parseRelationshipStatusOption(value: string): KloRelationshipStatusOption {
|
||||
if (value === 'accepted' || value === 'review' || value === 'rejected' || value === 'skipped' || value === 'all') {
|
||||
return value;
|
||||
}
|
||||
throw new InvalidArgumentError('Allowed choices are accepted, review, rejected, skipped, all');
|
||||
}
|
||||
|
||||
function parseRelationshipFeedbackDecisionOption(value: string): KloRelationshipFeedbackDecisionOption {
|
||||
if (value === 'accepted' || value === 'rejected' || value === 'all') {
|
||||
return value;
|
||||
}
|
||||
throw new InvalidArgumentError('Allowed choices are accepted, rejected, all');
|
||||
}
|
||||
|
||||
function parseNonEmptyOption(value: string): string {
|
||||
if (value.trim().length === 0) {
|
||||
throw new InvalidArgumentError('must not be empty');
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function parseRelationshipCalibrationThreshold(value: string): number {
|
||||
const parsed = Number(value);
|
||||
if (Number.isFinite(parsed) && parsed >= 0 && parsed <= 1) {
|
||||
return parsed;
|
||||
}
|
||||
throw new InvalidArgumentError('Allowed range is 0 through 1');
|
||||
}
|
||||
|
||||
function relationshipDecisionArgs(options: {
|
||||
accept?: string;
|
||||
reject?: string;
|
||||
reviewer?: string;
|
||||
note?: string;
|
||||
json?: boolean;
|
||||
}): Pick<
|
||||
Extract<KloScanArgs, { command: 'relationshipDecision' }>,
|
||||
'candidateId' | 'decision' | 'reviewer' | 'note' | 'json'
|
||||
> | null {
|
||||
const decisionCount = [options.accept !== undefined, options.reject !== undefined].filter(Boolean).length;
|
||||
if (decisionCount > 1) {
|
||||
throw new Error('Only one relationship review decision option can be used: --accept and --reject conflict');
|
||||
}
|
||||
if (options.accept !== undefined) {
|
||||
return {
|
||||
candidateId: options.accept,
|
||||
decision: 'accepted',
|
||||
reviewer: options.reviewer ?? 'klo',
|
||||
note: options.note ?? null,
|
||||
json: options.json === true,
|
||||
};
|
||||
}
|
||||
if (options.reject !== undefined) {
|
||||
return {
|
||||
candidateId: options.reject,
|
||||
decision: 'rejected',
|
||||
reviewer: options.reviewer ?? 'klo',
|
||||
note: options.note ?? null,
|
||||
json: options.json === true,
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function collectRelationshipCandidateOption(value: string, previous: string[]): string[] {
|
||||
return [...previous, parseNonEmptyOption(value)];
|
||||
}
|
||||
|
||||
export function registerScanCommands(program: Command, context: KloCliCommandContext): void {
|
||||
const scan = program
|
||||
.command('scan')
|
||||
.description('Run or inspect standalone connection scans')
|
||||
.argument('[connectionId]', 'KLO connection id to scan')
|
||||
.option(
|
||||
'--mode <mode>',
|
||||
'Scan mode: structural, enriched, relationships (default: structural)',
|
||||
parseScanModeOption,
|
||||
)
|
||||
.option('--dry-run', 'Run without writing scan results', false)
|
||||
.option('--database-introspection-url <url>', 'Daemon URL for live-database introspection')
|
||||
.showHelpAfterError()
|
||||
.addHelpText(
|
||||
'after',
|
||||
'\nProject directory defaults to KLO_PROJECT_DIR when set, otherwise the current working directory.\n',
|
||||
)
|
||||
.hook('preAction', (_thisCommand, actionCommand) => {
|
||||
context.writeDebug?.('scan', actionCommand);
|
||||
})
|
||||
.action(async (connectionId: string | undefined, options, command) => {
|
||||
if (!connectionId) {
|
||||
scan.outputHelp();
|
||||
context.io.stderr.write('klo dev scan requires <connectionId> or a subcommand\n');
|
||||
context.setExitCode(1);
|
||||
return;
|
||||
}
|
||||
const mode = options.mode ?? 'structural';
|
||||
await runScanArgs(context, {
|
||||
command: 'run',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
connectionId,
|
||||
mode,
|
||||
detectRelationships: mode === 'relationships',
|
||||
dryRun: options.dryRun === true,
|
||||
databaseIntrospectionUrl: options.databaseIntrospectionUrl,
|
||||
});
|
||||
});
|
||||
|
||||
scan
|
||||
.command('status')
|
||||
.description('Print status for a local scan run')
|
||||
.argument('<runId>', 'Local scan run id')
|
||||
.addHelpText(
|
||||
'after',
|
||||
'\n--project-dir is inherited from `klo dev scan` (default: KLO_PROJECT_DIR or current working directory).\n',
|
||||
)
|
||||
.action(async (runId: string, _options: unknown, command) => {
|
||||
await runScanArgs(context, {
|
||||
command: 'status',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
runId,
|
||||
});
|
||||
});
|
||||
|
||||
scan
|
||||
.command('report')
|
||||
.description('Print a local scan report')
|
||||
.argument('<runId>', 'Local scan run id')
|
||||
.option('--json', 'Print the raw scan report JSON', false)
|
||||
.addHelpText(
|
||||
'after',
|
||||
'\n--project-dir is inherited from `klo dev scan` (default: KLO_PROJECT_DIR or current working directory).\n',
|
||||
)
|
||||
.action(async (runId: string, options, command) => {
|
||||
await runScanArgs(context, {
|
||||
command: 'report',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
runId,
|
||||
json: options.json === true,
|
||||
});
|
||||
});
|
||||
|
||||
scan
|
||||
.command('relationships')
|
||||
.description('Print relationship artifacts for a local scan run')
|
||||
.argument('<runId>', 'Local scan run id')
|
||||
.option(
|
||||
'--status <status>',
|
||||
'Relationship status: accepted, review, rejected, skipped, all',
|
||||
parseRelationshipStatusOption,
|
||||
'review',
|
||||
)
|
||||
.option('--limit <count>', 'Maximum relationships to print per status', parsePositiveIntegerOption, 25)
|
||||
.addOption(
|
||||
new Option('--accept <candidateId>', 'Record a reviewer accepted decision for a relationship candidate')
|
||||
.argParser(parseNonEmptyOption)
|
||||
.conflicts('reject'),
|
||||
)
|
||||
.addOption(
|
||||
new Option('--reject <candidateId>', 'Record a reviewer rejected decision for a relationship candidate')
|
||||
.argParser(parseNonEmptyOption)
|
||||
.conflicts('accept'),
|
||||
)
|
||||
.option('--note <text>', 'Attach a note when recording a relationship review decision')
|
||||
.option('--reviewer <name>', 'Reviewer name for a relationship review decision')
|
||||
.option('--json', 'Print relationship artifacts as JSON', false)
|
||||
.addHelpText(
|
||||
'after',
|
||||
'\n--project-dir is inherited from `klo dev scan` (default: KLO_PROJECT_DIR or current working directory).\n',
|
||||
)
|
||||
.action(async (runId: string, options, command) => {
|
||||
const decision = relationshipDecisionArgs(options);
|
||||
if (decision) {
|
||||
await runScanArgs(context, {
|
||||
command: 'relationshipDecision',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
runId,
|
||||
candidateId: decision.candidateId,
|
||||
decision: decision.decision,
|
||||
reviewer: decision.reviewer,
|
||||
note: decision.note,
|
||||
json: decision.json,
|
||||
});
|
||||
return;
|
||||
}
|
||||
await runScanArgs(context, {
|
||||
command: 'relationships',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
runId,
|
||||
status: options.status,
|
||||
json: options.json === true,
|
||||
limit: options.limit,
|
||||
});
|
||||
});
|
||||
|
||||
scan
|
||||
.command('relationship-apply')
|
||||
.description('Apply accepted relationship review decisions as manual manifest joins')
|
||||
.argument('<runId>', 'Local scan run id')
|
||||
.option('--all-accepted', 'Apply all accepted relationship review decisions for the scan run', false)
|
||||
.option(
|
||||
'--candidate <candidateId>',
|
||||
'Apply one accepted relationship review decision',
|
||||
collectRelationshipCandidateOption,
|
||||
[],
|
||||
)
|
||||
.option('--dry-run', 'Preview relationships that would be written without rewriting manifest shards', false)
|
||||
.option('--json', 'Print the apply result as JSON', false)
|
||||
.addHelpText(
|
||||
'after',
|
||||
'\n--project-dir is inherited from `klo dev scan` (default: KLO_PROJECT_DIR or current working directory).\n',
|
||||
)
|
||||
.action(async (runId: string, options, command) => {
|
||||
const parentOptions = command.parent?.opts() as { dryRun?: boolean } | undefined;
|
||||
await runScanArgs(context, {
|
||||
command: 'relationshipApply',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
runId,
|
||||
applyAllAccepted: options.allAccepted === true,
|
||||
candidateIds: options.candidate,
|
||||
dryRun: options.dryRun === true || parentOptions?.dryRun === true,
|
||||
json: options.json === true,
|
||||
});
|
||||
});
|
||||
|
||||
scan
|
||||
.command('relationship-feedback')
|
||||
.description('Export persisted relationship review decisions as calibration labels')
|
||||
.option('--connection <connectionId>', 'Only export labels for one KLO connection')
|
||||
.option(
|
||||
'--decision <decision>',
|
||||
'Relationship feedback decision: accepted, rejected, all',
|
||||
parseRelationshipFeedbackDecisionOption,
|
||||
'all',
|
||||
)
|
||||
.addOption(new Option('--json', 'Print the export as JSON').default(false).conflicts('jsonl'))
|
||||
.addOption(new Option('--jsonl', 'Print labels as newline-delimited JSON').default(false).conflicts('json'))
|
||||
.addHelpText(
|
||||
'after',
|
||||
'\n--project-dir is inherited from `klo dev scan` (default: KLO_PROJECT_DIR or current working directory).\n',
|
||||
)
|
||||
.action(async (options, command) => {
|
||||
await runScanArgs(context, {
|
||||
command: 'relationshipFeedback',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
connectionId: options.connection ?? null,
|
||||
decision: options.decision,
|
||||
json: options.json === true,
|
||||
jsonl: options.jsonl === true,
|
||||
});
|
||||
});
|
||||
|
||||
scan
|
||||
.command('relationship-calibration')
|
||||
.description('Summarize relationship feedback labels against current score thresholds')
|
||||
.option('--connection <connectionId>', 'Only calibrate labels for one KLO connection')
|
||||
.option(
|
||||
'--decision <decision>',
|
||||
'Relationship feedback decision: accepted, rejected, all',
|
||||
parseRelationshipFeedbackDecisionOption,
|
||||
'all',
|
||||
)
|
||||
.option(
|
||||
'--accept-threshold <value>',
|
||||
'Score threshold treated as predicted accepted',
|
||||
parseRelationshipCalibrationThreshold,
|
||||
0.85,
|
||||
)
|
||||
.option(
|
||||
'--review-threshold <value>',
|
||||
'Score threshold treated as predicted review',
|
||||
parseRelationshipCalibrationThreshold,
|
||||
0.55,
|
||||
)
|
||||
.option('--json', 'Print the calibration report as JSON', false)
|
||||
.addHelpText(
|
||||
'after',
|
||||
'\n--project-dir is inherited from `klo dev scan` (default: KLO_PROJECT_DIR or current working directory).\n',
|
||||
)
|
||||
.action(async (options, command) => {
|
||||
await runScanArgs(context, {
|
||||
command: 'relationshipCalibration',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
connectionId: options.connection ?? null,
|
||||
decision: options.decision,
|
||||
acceptThreshold: options.acceptThreshold,
|
||||
reviewThreshold: options.reviewThreshold,
|
||||
json: options.json === true,
|
||||
});
|
||||
});
|
||||
|
||||
scan
|
||||
.command('relationship-thresholds')
|
||||
.description('Evaluate relationship feedback labels for offline threshold advice')
|
||||
.option('--connection <connectionId>', 'Only evaluate labels for one KLO connection')
|
||||
.option(
|
||||
'--min-total-labels <count>',
|
||||
'Minimum scored labels before advice can be ready',
|
||||
parsePositiveIntegerOption,
|
||||
20,
|
||||
)
|
||||
.option(
|
||||
'--min-accepted-labels <count>',
|
||||
'Minimum accepted labels before advice can be ready',
|
||||
parsePositiveIntegerOption,
|
||||
5,
|
||||
)
|
||||
.option(
|
||||
'--min-rejected-labels <count>',
|
||||
'Minimum rejected labels before advice can be ready',
|
||||
parsePositiveIntegerOption,
|
||||
5,
|
||||
)
|
||||
.option('--json', 'Print the threshold advice report as JSON', false)
|
||||
.addHelpText(
|
||||
'after',
|
||||
'\n--project-dir is inherited from `klo dev scan` (default: KLO_PROJECT_DIR or current working directory).\n',
|
||||
)
|
||||
.action(async (options, command) => {
|
||||
await runScanArgs(context, {
|
||||
command: 'relationshipThresholds',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
connectionId: options.connection ?? null,
|
||||
minTotalLabels: options.minTotalLabels,
|
||||
minAcceptedLabels: options.minAcceptedLabels,
|
||||
minRejectedLabels: options.minRejectedLabels,
|
||||
json: options.json === true,
|
||||
});
|
||||
});
|
||||
}
|
||||
47
packages/cli/src/commands/serve-commands.ts
Normal file
47
packages/cli/src/commands/serve-commands.ts
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
import { type Command, InvalidArgumentError } from '@commander-js/extra-typings';
|
||||
import { type KloCliCommandContext, resolveCommandProjectDir } from '../cli-program.js';
|
||||
import type { KloServeArgs } from '../serve.js';
|
||||
import { profileMark } from '../startup-profile.js';
|
||||
|
||||
profileMark('module:commands/serve-commands');
|
||||
|
||||
function parseMcp(value: string): 'stdio' {
|
||||
if (value === 'stdio') {
|
||||
return 'stdio';
|
||||
}
|
||||
throw new InvalidArgumentError('Only stdio is supported in this phase');
|
||||
}
|
||||
|
||||
export function registerServeCommands(program: Command, context: KloCliCommandContext): void {
|
||||
program
|
||||
.command('serve')
|
||||
.description('Run standalone KLO services such as MCP stdio')
|
||||
.requiredOption('--mcp <mode>', 'MCP transport mode', parseMcp)
|
||||
.option('--user-id <id>', 'Local user id', 'local')
|
||||
.option('--semantic-compute', 'Enable semantic-layer compute', false)
|
||||
.option('--semantic-compute-url <url>', 'HTTP semantic-layer compute URL')
|
||||
.option('--database-introspection-url <url>', 'Daemon URL for live-database introspection')
|
||||
.option('--execute-queries', 'Allow semantic-layer query execution', false)
|
||||
.option('--memory-capture', 'Enable memory capture', false)
|
||||
.option('--memory-model <model>', 'Memory capture model')
|
||||
.showHelpAfterError()
|
||||
.action(async (options, command): Promise<void> => {
|
||||
const semanticCompute = options.semanticCompute === true || Boolean(options.semanticComputeUrl);
|
||||
if (options.executeQueries === true && !semanticCompute) {
|
||||
throw new Error('--execute-queries requires --semantic-compute');
|
||||
}
|
||||
const args: KloServeArgs = {
|
||||
mcp: options.mcp,
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
userId: options.userId,
|
||||
semanticCompute,
|
||||
semanticComputeUrl: options.semanticComputeUrl,
|
||||
databaseIntrospectionUrl: options.databaseIntrospectionUrl,
|
||||
executeQueries: options.executeQueries === true,
|
||||
memoryCapture: options.memoryCapture === true,
|
||||
memoryModel: options.memoryModel,
|
||||
};
|
||||
const runner = context.deps.serveStdio ?? (await import('../serve.js')).runKloServeStdio;
|
||||
context.setExitCode(await runner(args));
|
||||
});
|
||||
}
|
||||
517
packages/cli/src/commands/setup-commands.ts
Normal file
517
packages/cli/src/commands/setup-commands.ts
Normal file
|
|
@ -0,0 +1,517 @@
|
|||
import { type Command, InvalidArgumentError, Option } from '@commander-js/extra-typings';
|
||||
import type { KloCliCommandContext } from '../cli-program.js';
|
||||
import { resolveCommandProjectDir } from '../cli-program.js';
|
||||
import type { KloSetupDatabaseDriver } from '../setup-databases.js';
|
||||
import type { KloSetupSourceType } from '../setup-sources.js';
|
||||
import { registerDemoCommands } from './demo-commands.js';
|
||||
|
||||
async function runSetupArgs(
|
||||
context: KloCliCommandContext,
|
||||
args: Parameters<NonNullable<typeof context.deps.setup>>[0],
|
||||
) {
|
||||
const runner = context.deps.setup ?? (await import('../setup.js')).runKloSetup;
|
||||
context.setExitCode(await runner(args, context.io));
|
||||
}
|
||||
|
||||
function positiveInteger(value: string): number {
|
||||
const parsed = Number.parseInt(value, 10);
|
||||
if (!Number.isInteger(parsed) || parsed <= 0) {
|
||||
throw new Error(`Expected a positive integer, received ${value}`);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function embeddingBackend(value: string): 'openai' | 'sentence-transformers' {
|
||||
if (value === 'openai' || value === 'sentence-transformers') {
|
||||
return value;
|
||||
}
|
||||
throw new InvalidArgumentError(`invalid choice '${value}'`);
|
||||
}
|
||||
|
||||
function databaseDriver(value: string): KloSetupDatabaseDriver {
|
||||
if (
|
||||
value === 'sqlite' ||
|
||||
value === 'postgres' ||
|
||||
value === 'mysql' ||
|
||||
value === 'clickhouse' ||
|
||||
value === 'sqlserver' ||
|
||||
value === 'bigquery' ||
|
||||
value === 'snowflake'
|
||||
) {
|
||||
return value;
|
||||
}
|
||||
throw new InvalidArgumentError(`invalid choice '${value}'`);
|
||||
}
|
||||
|
||||
function sourceType(value: string): KloSetupSourceType {
|
||||
if (
|
||||
value === 'dbt' ||
|
||||
value === 'metricflow' ||
|
||||
value === 'metabase' ||
|
||||
value === 'looker' ||
|
||||
value === 'lookml' ||
|
||||
value === 'notion'
|
||||
) {
|
||||
return value;
|
||||
}
|
||||
throw new InvalidArgumentError(`invalid choice '${value}'`);
|
||||
}
|
||||
|
||||
function agentScope(value: string): 'project' | 'global' {
|
||||
if (value === 'project' || value === 'global') {
|
||||
return value;
|
||||
}
|
||||
throw new InvalidArgumentError(`invalid choice '${value}'`);
|
||||
}
|
||||
|
||||
function agentInstallMode(value: string): 'cli' | 'mcp' | 'both' {
|
||||
if (value === 'cli' || value === 'mcp' || value === 'both') {
|
||||
return value;
|
||||
}
|
||||
throw new InvalidArgumentError(`invalid choice '${value}'`);
|
||||
}
|
||||
|
||||
function positiveNumber(value: string): number {
|
||||
const parsed = Number.parseInt(value, 10);
|
||||
if (!Number.isInteger(parsed) || parsed <= 0) {
|
||||
throw new InvalidArgumentError(`Expected a positive integer, received ${value}`);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function optionWasSpecified(command: Command, optionName: string): boolean {
|
||||
const commandWithSources = command as Command & {
|
||||
getOptionValueSource?: (name: string) => string | undefined;
|
||||
getOptionValueSourceWithGlobals?: (name: string) => string | undefined;
|
||||
};
|
||||
const source =
|
||||
commandWithSources.getOptionValueSourceWithGlobals?.(optionName) ??
|
||||
commandWithSources.getOptionValueSource?.(optionName);
|
||||
return source !== undefined && source !== 'default';
|
||||
}
|
||||
|
||||
function shouldShowSetupEntryMenu(
|
||||
options: {
|
||||
new?: boolean;
|
||||
existing?: boolean;
|
||||
agents?: boolean;
|
||||
target?: string;
|
||||
global?: boolean;
|
||||
project?: boolean;
|
||||
skipAgents?: boolean;
|
||||
yes?: boolean;
|
||||
input?: boolean;
|
||||
anthropicApiKeyEnv?: string;
|
||||
anthropicApiKeyFile?: string;
|
||||
anthropicModel?: string;
|
||||
skipLlm?: boolean;
|
||||
embeddingBackend?: string;
|
||||
embeddingApiKeyEnv?: string;
|
||||
embeddingApiKeyFile?: string;
|
||||
skipEmbeddings?: boolean;
|
||||
database?: KloSetupDatabaseDriver[];
|
||||
databaseConnectionId?: string[];
|
||||
newDatabaseConnectionId?: string;
|
||||
databaseUrl?: string;
|
||||
databaseSchema?: string[];
|
||||
enableHistoricSql?: boolean;
|
||||
disableHistoricSql?: boolean;
|
||||
historicSqlWindowDays?: number;
|
||||
historicSqlMinCalls?: number;
|
||||
historicSqlServiceAccountPattern?: string[];
|
||||
historicSqlRedactionPattern?: string[];
|
||||
skipDatabases?: boolean;
|
||||
source?: KloSetupSourceType;
|
||||
sourceConnectionId?: string;
|
||||
sourcePath?: string;
|
||||
sourceGitUrl?: string;
|
||||
sourceBranch?: string;
|
||||
sourceSubpath?: string;
|
||||
sourceAuthTokenRef?: string;
|
||||
sourceUrl?: string;
|
||||
sourceApiKeyRef?: string;
|
||||
sourceClientId?: string;
|
||||
sourceClientSecretRef?: string;
|
||||
sourceWarehouseConnectionId?: string;
|
||||
sourceProjectName?: string;
|
||||
sourceProfilesPath?: string;
|
||||
sourceTarget?: string;
|
||||
metabaseDatabaseId?: number;
|
||||
notionCrawlMode?: string;
|
||||
notionRootPageId?: string[];
|
||||
skipInitialSourceIngest?: boolean;
|
||||
skipSources?: boolean;
|
||||
},
|
||||
command: Command,
|
||||
): boolean {
|
||||
if (options.database && options.database.length > 0) {
|
||||
return false;
|
||||
}
|
||||
if (options.databaseConnectionId && options.databaseConnectionId.length > 0) {
|
||||
return false;
|
||||
}
|
||||
if (options.databaseSchema && options.databaseSchema.length > 0) {
|
||||
return false;
|
||||
}
|
||||
if (options.historicSqlServiceAccountPattern && options.historicSqlServiceAccountPattern.length > 0) {
|
||||
return false;
|
||||
}
|
||||
if (options.historicSqlRedactionPattern && options.historicSqlRedactionPattern.length > 0) {
|
||||
return false;
|
||||
}
|
||||
if (options.notionRootPageId && options.notionRootPageId.length > 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return ![
|
||||
'new',
|
||||
'existing',
|
||||
'agents',
|
||||
'target',
|
||||
'global',
|
||||
'project',
|
||||
'skipAgents',
|
||||
'yes',
|
||||
'input',
|
||||
'anthropicApiKeyEnv',
|
||||
'anthropicApiKeyFile',
|
||||
'anthropicModel',
|
||||
'skipLlm',
|
||||
'embeddingBackend',
|
||||
'embeddingApiKeyEnv',
|
||||
'embeddingApiKeyFile',
|
||||
'skipEmbeddings',
|
||||
'newDatabaseConnectionId',
|
||||
'databaseUrl',
|
||||
'enableHistoricSql',
|
||||
'disableHistoricSql',
|
||||
'historicSqlWindowDays',
|
||||
'historicSqlMinCalls',
|
||||
'skipDatabases',
|
||||
'source',
|
||||
'sourceConnectionId',
|
||||
'sourcePath',
|
||||
'sourceGitUrl',
|
||||
'sourceBranch',
|
||||
'sourceSubpath',
|
||||
'sourceAuthTokenRef',
|
||||
'sourceUrl',
|
||||
'sourceApiKeyRef',
|
||||
'sourceClientId',
|
||||
'sourceClientSecretRef',
|
||||
'sourceWarehouseConnectionId',
|
||||
'sourceProjectName',
|
||||
'sourceProfilesPath',
|
||||
'sourceTarget',
|
||||
'metabaseDatabaseId',
|
||||
'notionCrawlMode',
|
||||
'skipInitialSourceIngest',
|
||||
'skipSources',
|
||||
].some((optionName) => optionWasSpecified(command, optionName));
|
||||
}
|
||||
|
||||
export function registerSetupCommands(program: Command, context: KloCliCommandContext): void {
|
||||
const setup = program
|
||||
.command('setup')
|
||||
.description('Set up or resume a local KLO project')
|
||||
.option('--project-dir <path>', 'KLO project directory')
|
||||
.option('--new', 'Create a new KLO project before setup', false)
|
||||
.option('--existing', 'Use an existing KLO project', false)
|
||||
.option('--agents', 'Install agent integration only', false)
|
||||
.addOption(
|
||||
new Option('--target <target>', 'Agent target').choices([
|
||||
'claude-code',
|
||||
'codex',
|
||||
'cursor',
|
||||
'opencode',
|
||||
'universal',
|
||||
]),
|
||||
)
|
||||
.addOption(new Option('--agent-scope <scope>', 'Agent install scope').argParser(agentScope).default('project'))
|
||||
.option('--project', 'Install agent integration into the project scope', false)
|
||||
.option('--global', 'Install agent integration into the global target scope', false)
|
||||
.addOption(
|
||||
new Option('--agent-install-mode <mode>', 'Agent install mode').argParser(agentInstallMode).default('cli'),
|
||||
)
|
||||
.option('--skip-agents', 'Leave agent integration incomplete for now', false)
|
||||
.option('--yes', 'Accept safe defaults in non-interactive setup', false)
|
||||
.option('--no-input', 'Disable interactive terminal input')
|
||||
.option('--anthropic-api-key-env <name>', 'Environment variable containing the Anthropic API key')
|
||||
.option('--anthropic-api-key-file <path>', 'File containing the Anthropic API key')
|
||||
.option('--anthropic-model <model>', 'Anthropic model ID to validate and save')
|
||||
.addOption(new Option('--skip-llm', 'Leave LLM setup incomplete for now').hideHelp().default(false))
|
||||
.addOption(new Option('--embedding-backend <backend>', 'Embedding backend').argParser(embeddingBackend))
|
||||
.option('--embedding-api-key-env <name>', 'Environment variable containing the embedding provider API key')
|
||||
.option('--embedding-api-key-file <path>', 'File containing the embedding provider API key')
|
||||
.addOption(new Option('--skip-embeddings', 'Leave embedding setup incomplete for now').hideHelp().default(false))
|
||||
.option(
|
||||
'--database <driver>',
|
||||
'Database driver to configure; repeatable',
|
||||
(value, previous: KloSetupDatabaseDriver[]) => {
|
||||
return [...previous, databaseDriver(value)];
|
||||
},
|
||||
[] as KloSetupDatabaseDriver[],
|
||||
)
|
||||
.option(
|
||||
'--database-connection-id <id>',
|
||||
'Existing selected connection id or new connection id',
|
||||
(value, previous: string[]) => [...previous, value],
|
||||
[],
|
||||
)
|
||||
.option('--new-database-connection-id <id>', 'Connection id for one new database connection', (value) => {
|
||||
if (!/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(value)) {
|
||||
throw new InvalidArgumentError(`Unsafe connection id: ${value}`);
|
||||
}
|
||||
return value;
|
||||
})
|
||||
.option('--database-url <url>', 'URL, env:NAME, or file:/path for one new URL-style database connection')
|
||||
.option(
|
||||
'--database-schema <schema>',
|
||||
'Database schema to include; repeatable',
|
||||
(value, previous: string[]) => [...previous, value],
|
||||
[],
|
||||
)
|
||||
.option('--enable-historic-sql', 'Enable Historic SQL when the selected database supports it', false)
|
||||
.option('--disable-historic-sql', 'Disable Historic SQL for the selected database', false)
|
||||
.option('--historic-sql-window-days <number>', 'Historic SQL query-history window', positiveInteger)
|
||||
.option(
|
||||
'--historic-sql-min-calls <number>',
|
||||
'Postgres Historic SQL pg_stat_statements minimum calls floor',
|
||||
positiveInteger,
|
||||
)
|
||||
.option(
|
||||
'--historic-sql-service-account-pattern <pattern>',
|
||||
'Historic SQL service-account regex; repeatable',
|
||||
(value, previous: string[]) => [...previous, value],
|
||||
[],
|
||||
)
|
||||
.option(
|
||||
'--historic-sql-redaction-pattern <pattern>',
|
||||
'Historic SQL SQL-literal redaction regex; repeatable',
|
||||
(value, previous: string[]) => [...previous, value],
|
||||
[],
|
||||
)
|
||||
.option('--skip-databases', 'Leave database setup incomplete; KLO cannot work until a primary source is added', false)
|
||||
.addOption(new Option('--source <type>', 'Source connector type').argParser(sourceType))
|
||||
.option('--source-connection-id <id>', 'Connection id for source setup')
|
||||
.option('--source-path <path>', 'Local source path for dbt, MetricFlow, or LookML')
|
||||
.option('--source-git-url <url>', 'Git URL for dbt, MetricFlow, or LookML')
|
||||
.option('--source-branch <branch>', 'Git branch for source setup')
|
||||
.option('--source-subpath <path>', 'Repo subpath for source setup')
|
||||
.option('--source-auth-token-ref <ref>', 'env: or file: credential ref for source repo auth')
|
||||
.option('--source-url <url>', 'Source service URL for Metabase or Looker')
|
||||
.option('--source-api-key-ref <ref>', 'env: or file: API key ref for Metabase or Notion')
|
||||
.option('--source-client-id <id>', 'Looker client id')
|
||||
.option('--source-client-secret-ref <ref>', 'env: or file: Looker client secret ref')
|
||||
.option('--source-warehouse-connection-id <id>', 'Mapped warehouse connection id')
|
||||
.option('--source-project-name <name>', 'dbt project name override')
|
||||
.option('--source-profiles-path <path>', 'dbt profiles path')
|
||||
.option('--source-target <target>', 'dbt target or source-specific mapping target')
|
||||
.option('--metabase-database-id <id>', 'Metabase database id to map', positiveNumber)
|
||||
.addOption(
|
||||
new Option('--notion-crawl-mode <mode>', 'Notion crawl mode').choices(['all_accessible', 'selected_roots']),
|
||||
)
|
||||
.option(
|
||||
'--notion-root-page-id <id>',
|
||||
'Notion root page id; repeatable',
|
||||
(value, previous: string[]) => [...previous, value],
|
||||
[],
|
||||
)
|
||||
.option('--skip-initial-source-ingest', 'Validate source setup without building source context during setup', false)
|
||||
.option('--skip-sources', 'Mark optional source setup complete with no sources', false)
|
||||
.showHelpAfterError();
|
||||
|
||||
setup.hook('preAction', (_thisCommand, actionCommand) => {
|
||||
context.writeDebug?.('setup', actionCommand);
|
||||
});
|
||||
|
||||
setup.action(async (options, command) => {
|
||||
if (options.anthropicApiKeyEnv && options.anthropicApiKeyFile) {
|
||||
context.io.stderr.write(
|
||||
'Choose only one Anthropic credential source: --anthropic-api-key-env or --anthropic-api-key-file.\n',
|
||||
);
|
||||
context.setExitCode(1);
|
||||
return;
|
||||
}
|
||||
if (options.embeddingApiKeyEnv && options.embeddingApiKeyFile) {
|
||||
context.io.stderr.write(
|
||||
'Choose only one embedding credential source: --embedding-api-key-env or --embedding-api-key-file.\n',
|
||||
);
|
||||
context.setExitCode(1);
|
||||
return;
|
||||
}
|
||||
if (options.enableHistoricSql && options.disableHistoricSql) {
|
||||
context.io.stderr.write(
|
||||
'Choose only one Historic SQL action: --enable-historic-sql or --disable-historic-sql.\n',
|
||||
);
|
||||
context.setExitCode(1);
|
||||
return;
|
||||
}
|
||||
if (options.sourcePath && options.sourceGitUrl) {
|
||||
context.io.stderr.write('Choose only one source location: --source-path or --source-git-url.\n');
|
||||
context.setExitCode(1);
|
||||
return;
|
||||
}
|
||||
if (options.skipSources && options.source) {
|
||||
context.io.stderr.write('Choose either --source or --skip-sources.\n');
|
||||
context.setExitCode(1);
|
||||
return;
|
||||
}
|
||||
|
||||
const mode = options.new ? 'new' : options.existing ? 'existing' : 'auto';
|
||||
const resolvedAgentScope = options.global ? 'global' : options.agentScope;
|
||||
await runSetupArgs(context, {
|
||||
command: 'run',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
mode,
|
||||
agents: options.agents === true,
|
||||
...(options.target ? { target: options.target } : {}),
|
||||
agentScope: resolvedAgentScope,
|
||||
agentInstallMode: options.agentInstallMode,
|
||||
skipAgents: options.skipAgents === true,
|
||||
inputMode: options.input === false ? 'disabled' : 'auto',
|
||||
yes: options.yes === true,
|
||||
...(options.anthropicApiKeyEnv ? { anthropicApiKeyEnv: options.anthropicApiKeyEnv } : {}),
|
||||
...(options.anthropicApiKeyFile ? { anthropicApiKeyFile: options.anthropicApiKeyFile } : {}),
|
||||
...(options.anthropicModel ? { anthropicModel: options.anthropicModel } : {}),
|
||||
skipLlm: options.skipLlm === true,
|
||||
...(options.embeddingBackend ? { embeddingBackend: options.embeddingBackend } : {}),
|
||||
...(options.embeddingApiKeyEnv ? { embeddingApiKeyEnv: options.embeddingApiKeyEnv } : {}),
|
||||
...(options.embeddingApiKeyFile ? { embeddingApiKeyFile: options.embeddingApiKeyFile } : {}),
|
||||
skipEmbeddings: options.skipEmbeddings === true,
|
||||
...(options.database.length > 0 ? { databaseDrivers: options.database } : {}),
|
||||
...(options.databaseConnectionId.length > 0 ? { databaseConnectionIds: options.databaseConnectionId } : {}),
|
||||
...(options.newDatabaseConnectionId ? { databaseConnectionId: options.newDatabaseConnectionId } : {}),
|
||||
...(options.databaseUrl ? { databaseUrl: options.databaseUrl } : {}),
|
||||
databaseSchemas: options.databaseSchema,
|
||||
...(options.enableHistoricSql ? { enableHistoricSql: true } : {}),
|
||||
...(options.disableHistoricSql ? { disableHistoricSql: true } : {}),
|
||||
...(options.historicSqlWindowDays !== undefined ? { historicSqlWindowDays: options.historicSqlWindowDays } : {}),
|
||||
...(options.historicSqlMinCalls !== undefined ? { historicSqlMinCalls: options.historicSqlMinCalls } : {}),
|
||||
...(options.historicSqlServiceAccountPattern.length > 0
|
||||
? { historicSqlServiceAccountPatterns: options.historicSqlServiceAccountPattern }
|
||||
: {}),
|
||||
...(options.historicSqlRedactionPattern.length > 0
|
||||
? { historicSqlRedactionPatterns: options.historicSqlRedactionPattern }
|
||||
: {}),
|
||||
skipDatabases: options.skipDatabases === true,
|
||||
...(options.source ? { source: options.source } : {}),
|
||||
...(options.sourceConnectionId ? { sourceConnectionId: options.sourceConnectionId } : {}),
|
||||
...(options.sourcePath ? { sourcePath: options.sourcePath } : {}),
|
||||
...(options.sourceGitUrl ? { sourceGitUrl: options.sourceGitUrl } : {}),
|
||||
...(options.sourceBranch ? { sourceBranch: options.sourceBranch } : {}),
|
||||
...(options.sourceSubpath ? { sourceSubpath: options.sourceSubpath } : {}),
|
||||
...(options.sourceAuthTokenRef ? { sourceAuthTokenRef: options.sourceAuthTokenRef } : {}),
|
||||
...(options.sourceUrl ? { sourceUrl: options.sourceUrl } : {}),
|
||||
...(options.sourceApiKeyRef ? { sourceApiKeyRef: options.sourceApiKeyRef } : {}),
|
||||
...(options.sourceClientId ? { sourceClientId: options.sourceClientId } : {}),
|
||||
...(options.sourceClientSecretRef ? { sourceClientSecretRef: options.sourceClientSecretRef } : {}),
|
||||
...(options.sourceWarehouseConnectionId
|
||||
? { sourceWarehouseConnectionId: options.sourceWarehouseConnectionId }
|
||||
: {}),
|
||||
...(options.sourceProjectName ? { sourceProjectName: options.sourceProjectName } : {}),
|
||||
...(options.sourceProfilesPath ? { sourceProfilesPath: options.sourceProfilesPath } : {}),
|
||||
...(options.sourceTarget ? { sourceTarget: options.sourceTarget } : {}),
|
||||
...(options.metabaseDatabaseId !== undefined ? { metabaseDatabaseId: options.metabaseDatabaseId } : {}),
|
||||
...(options.notionCrawlMode ? { notionCrawlMode: options.notionCrawlMode } : {}),
|
||||
...(options.notionRootPageId.length > 0 ? { notionRootPageIds: options.notionRootPageId } : {}),
|
||||
runInitialSourceIngest: false,
|
||||
skipSources: options.skipSources === true,
|
||||
showEntryMenu: shouldShowSetupEntryMenu(options, command),
|
||||
});
|
||||
});
|
||||
|
||||
registerDemoCommands(setup, context, { description: 'Run the packaged KLO demo from setup' });
|
||||
|
||||
const setupContext = setup.command('context').description('Build, inspect, and recover setup-managed KLO context');
|
||||
|
||||
function setupContextInputMode(command: {
|
||||
optsWithGlobals?: () => unknown;
|
||||
opts?: () => unknown;
|
||||
}): 'auto' | 'disabled' {
|
||||
const options = command.optsWithGlobals?.() as { input?: boolean } | undefined;
|
||||
return options?.input === false ? 'disabled' : 'auto';
|
||||
}
|
||||
|
||||
setupContext
|
||||
.command('build')
|
||||
.description('Build agent-ready KLO context for setup')
|
||||
.option('--no-input', 'Disable interactive terminal input')
|
||||
.action(async (options: { input?: boolean }, command) => {
|
||||
await runSetupArgs(context, {
|
||||
command: 'context-build',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
inputMode: options.input === false ? 'disabled' : setupContextInputMode(command),
|
||||
});
|
||||
});
|
||||
|
||||
setupContext
|
||||
.command('watch')
|
||||
.description('Watch a setup-managed context build')
|
||||
.argument('[runId]', 'Setup context build run id')
|
||||
.option('--no-input', 'Disable interactive terminal input')
|
||||
.action(async (runId: string | undefined, options: { input?: boolean }, command) => {
|
||||
await runSetupArgs(context, {
|
||||
command: 'context-watch',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
...(runId ? { runId } : {}),
|
||||
inputMode: options.input === false ? 'disabled' : setupContextInputMode(command),
|
||||
});
|
||||
});
|
||||
|
||||
setupContext
|
||||
.command('status')
|
||||
.description('Print setup-managed context build status')
|
||||
.argument('[runId]', 'Setup context build run id')
|
||||
.option('--json', 'Print JSON output', false)
|
||||
.action(async (runId: string | undefined, options: { json?: boolean }, command) => {
|
||||
await runSetupArgs(context, {
|
||||
command: 'context-status',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
...(runId ? { runId } : {}),
|
||||
json: options.json === true,
|
||||
});
|
||||
});
|
||||
|
||||
setupContext
|
||||
.command('stop')
|
||||
.description('Request a pause for a setup-managed context build')
|
||||
.argument('[runId]', 'Setup context build run id')
|
||||
.option('--force', 'Request the pause without an interactive confirmation', false)
|
||||
.action(async (runId: string | undefined, _options: { force?: boolean }, command) => {
|
||||
await runSetupArgs(context, {
|
||||
command: 'context-stop',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
...(runId ? { runId } : {}),
|
||||
});
|
||||
});
|
||||
|
||||
setup
|
||||
.command('remove')
|
||||
.description('Remove setup-managed local integrations')
|
||||
.option('--agents', 'Remove setup-managed agent integration files', false)
|
||||
.action(async (options: { agents?: boolean }, command) => {
|
||||
const parentOptions = command.parent?.opts() as { agents?: boolean } | undefined;
|
||||
if (options.agents !== true && parentOptions?.agents !== true) {
|
||||
context.io.stderr.write('Choose what to remove: --agents.\n');
|
||||
context.setExitCode(1);
|
||||
return;
|
||||
}
|
||||
await runSetupArgs(context, {
|
||||
command: 'remove-agents',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
});
|
||||
});
|
||||
|
||||
setup
|
||||
.command('status')
|
||||
.description('Show setup readiness for the resolved KLO project')
|
||||
.option('--json', 'Print JSON output', false)
|
||||
.action(async (options: { json?: boolean }, command) => {
|
||||
await runSetupArgs(context, {
|
||||
command: 'status',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
json: options.json === true,
|
||||
});
|
||||
});
|
||||
}
|
||||
148
packages/cli/src/commands/sl-commands.ts
Normal file
148
packages/cli/src/commands/sl-commands.ts
Normal file
|
|
@ -0,0 +1,148 @@
|
|||
import { type Command, InvalidArgumentError, Option } from '@commander-js/extra-typings';
|
||||
import {
|
||||
collectOption,
|
||||
type KloCliCommandContext,
|
||||
parsePositiveIntegerOption,
|
||||
resolveCommandProjectDir,
|
||||
} from '../cli-program.js';
|
||||
import { slQueryCommandSchema } from '../command-schemas.js';
|
||||
import type { KloSlArgs } from '../sl.js';
|
||||
import { profileMark } from '../startup-profile.js';
|
||||
|
||||
profileMark('module:commands/sl-commands');
|
||||
|
||||
function parseOrderBy(value: string): string | { field: string; direction?: string } {
|
||||
const [field, direction] = value.split(':');
|
||||
if (!field) {
|
||||
throw new InvalidArgumentError('requires a field');
|
||||
}
|
||||
if (!direction) {
|
||||
return field;
|
||||
}
|
||||
if (direction !== 'asc' && direction !== 'desc') {
|
||||
throw new InvalidArgumentError('direction must be asc or desc');
|
||||
}
|
||||
return { field, direction };
|
||||
}
|
||||
|
||||
function collectOrderBy(
|
||||
value: string,
|
||||
previous: Array<string | { field: string; direction?: string }> = [],
|
||||
): Array<string | { field: string; direction?: string }> {
|
||||
return [...previous, parseOrderBy(value)];
|
||||
}
|
||||
|
||||
async function runSlArgs(context: KloCliCommandContext, args: KloSlArgs): Promise<void> {
|
||||
const runner = context.deps.sl ?? (await import('../sl.js')).runKloSl;
|
||||
context.setExitCode(await runner(args, context.io));
|
||||
}
|
||||
|
||||
export function registerSlCommands(program: Command, context: KloCliCommandContext, commandName = 'sl'): void {
|
||||
const sl = program
|
||||
.command(commandName)
|
||||
.description('List, read, validate, query, or write local semantic-layer sources')
|
||||
.showHelpAfterError()
|
||||
.addHelpText(
|
||||
'after',
|
||||
'\nProject directory defaults to KLO_PROJECT_DIR when set, otherwise the current working directory.\n',
|
||||
);
|
||||
|
||||
sl.command('list')
|
||||
.description('List semantic-layer sources')
|
||||
.option('--connection-id <id>', 'KLO connection id')
|
||||
.addOption(
|
||||
new Option('--output <mode>', 'Output mode: pretty (default in TTY), plain (TSV), or json').choices([
|
||||
'pretty',
|
||||
'plain',
|
||||
'json',
|
||||
]),
|
||||
)
|
||||
.option('--json', 'Shortcut for --output=json (overrides --output)', false)
|
||||
.action(async (options: { connectionId?: string; output?: 'pretty' | 'plain' | 'json'; json?: boolean }, command) => {
|
||||
await runSlArgs(context, {
|
||||
command: 'list',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
connectionId: options.connectionId,
|
||||
output: options.output,
|
||||
json: options.json,
|
||||
});
|
||||
});
|
||||
|
||||
sl.command('read')
|
||||
.description('Read a semantic-layer source')
|
||||
.argument('<sourceName>', 'Semantic-layer source name')
|
||||
.requiredOption('--connection-id <id>', 'KLO connection id')
|
||||
.action(async (sourceName: string, options: { connectionId: string }, command) => {
|
||||
await runSlArgs(context, {
|
||||
command: 'read',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
connectionId: options.connectionId,
|
||||
sourceName,
|
||||
});
|
||||
});
|
||||
|
||||
sl.command('validate')
|
||||
.description('Validate a semantic-layer source')
|
||||
.argument('<sourceName>', 'Semantic-layer source name')
|
||||
.requiredOption('--connection-id <id>', 'KLO connection id')
|
||||
.action(async (sourceName: string, options: { connectionId: string }, command) => {
|
||||
await runSlArgs(context, {
|
||||
command: 'validate',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
connectionId: options.connectionId,
|
||||
sourceName,
|
||||
});
|
||||
});
|
||||
|
||||
sl.command('write')
|
||||
.description('Write a semantic-layer source')
|
||||
.argument('<sourceName>', 'Semantic-layer source name')
|
||||
.requiredOption('--connection-id <id>', 'KLO connection id')
|
||||
.requiredOption('--yaml <yaml>', 'Semantic-layer source YAML')
|
||||
.action(async (sourceName: string, options: { connectionId: string; yaml: string }, command) => {
|
||||
await runSlArgs(context, {
|
||||
command: 'write',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
connectionId: options.connectionId,
|
||||
sourceName,
|
||||
yaml: options.yaml,
|
||||
});
|
||||
});
|
||||
|
||||
sl.command('query')
|
||||
.description('Compile or execute a semantic-layer query')
|
||||
.option('--connection-id <id>', 'KLO connection id')
|
||||
.option('--measure <measure>', 'Measure to query; repeatable', collectOption, [])
|
||||
.option('--dimension <dimension>', 'Dimension to include; repeatable', collectOption, [])
|
||||
.option('--filter <filter>', 'Filter expression; repeatable', collectOption, [])
|
||||
.option('--segment <segment>', 'Segment to include; repeatable', collectOption, [])
|
||||
.option('--order-by <field[:direction]>', 'Order field, optionally suffixed with :asc or :desc', collectOrderBy, [])
|
||||
.option('--limit <n>', 'Query limit', parsePositiveIntegerOption)
|
||||
.option('--include-empty', 'Include empty rows', false)
|
||||
.addOption(new Option('--format <format>', 'json or sql').choices(['json', 'sql']).default('json'))
|
||||
.option('--execute', 'Execute the compiled query', false)
|
||||
.option('--max-rows <n>', 'Maximum rows to return when executing', parsePositiveIntegerOption)
|
||||
.action(async (options, command) => {
|
||||
if (options.measure.length === 0) {
|
||||
throw new Error('sl query requires at least one --measure');
|
||||
}
|
||||
const args = slQueryCommandSchema.parse({
|
||||
command: 'query',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
connectionId: options.connectionId,
|
||||
query: {
|
||||
measures: options.measure,
|
||||
dimensions: options.dimension,
|
||||
...(options.filter.length > 0 ? { filters: options.filter } : {}),
|
||||
...(options.segment.length > 0 ? { segments: options.segment } : {}),
|
||||
...(options.orderBy.length > 0 ? { order_by: options.orderBy } : {}),
|
||||
...(options.limit !== undefined ? { limit: options.limit } : {}),
|
||||
...(options.includeEmpty === true ? { include_empty: true } : {}),
|
||||
},
|
||||
format: options.format,
|
||||
execute: options.execute === true,
|
||||
...(options.maxRows !== undefined ? { maxRows: options.maxRows } : {}),
|
||||
});
|
||||
await runSlArgs(context, args);
|
||||
});
|
||||
}
|
||||
23
packages/cli/src/commands/status-commands.ts
Normal file
23
packages/cli/src/commands/status-commands.ts
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
import type { Command } from '@commander-js/extra-typings';
|
||||
import type { KloCliCommandContext } from '../cli-program.js';
|
||||
import { resolveCommandProjectDir } from '../cli-program.js';
|
||||
|
||||
export function registerStatusCommands(program: Command, context: KloCliCommandContext): void {
|
||||
program
|
||||
.command('status')
|
||||
.description('Show current KLO project setup status')
|
||||
.option('--json', 'Print JSON output', false)
|
||||
.action(async (options: { json?: boolean }, command) => {
|
||||
const runner = context.deps.setup ?? (await import('../setup.js')).runKloSetup;
|
||||
context.setExitCode(
|
||||
await runner(
|
||||
{
|
||||
command: 'status',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
json: options.json === true,
|
||||
},
|
||||
context.io,
|
||||
),
|
||||
);
|
||||
});
|
||||
}
|
||||
353
packages/cli/src/completion.ts
Normal file
353
packages/cli/src/completion.ts
Normal file
|
|
@ -0,0 +1,353 @@
|
|||
import { mkdir, readFile, writeFile } from 'node:fs/promises';
|
||||
import { homedir } from 'node:os';
|
||||
import { dirname, join } from 'node:path';
|
||||
import type { CommandUnknownOpts, Option } from '@commander-js/extra-typings';
|
||||
|
||||
export interface CompletionRequest {
|
||||
position: number;
|
||||
words: string[];
|
||||
}
|
||||
|
||||
interface CompletionCandidate {
|
||||
value: string;
|
||||
description?: string;
|
||||
}
|
||||
|
||||
interface CommandWithHiddenFlag extends CommandUnknownOpts {
|
||||
_hidden?: boolean;
|
||||
}
|
||||
|
||||
interface ResolveState {
|
||||
command: CommandUnknownOpts;
|
||||
pendingOption?: Option;
|
||||
positionalIndex: number;
|
||||
}
|
||||
|
||||
export interface ZshCompletionInstallResult {
|
||||
completionPath: string;
|
||||
zshrcPath: string;
|
||||
}
|
||||
|
||||
const KLO_COMPLETION_BLOCK_START = '# >>> klo completion >>>';
|
||||
const KLO_COMPLETION_BLOCK_END = '# <<< klo completion <<<';
|
||||
const KLO_COMPLETION_BLOCK_PATTERN = new RegExp(
|
||||
`\\n?${escapeRegExp(KLO_COMPLETION_BLOCK_START)}[\\s\\S]*?${escapeRegExp(KLO_COMPLETION_BLOCK_END)}\\n?`,
|
||||
'g',
|
||||
);
|
||||
|
||||
export function zshCompletionScript(): string {
|
||||
const zshWords = '$' + '{words[@]}';
|
||||
const zshCompletionCapture = [
|
||||
'$',
|
||||
`{(@f)$("${'$'}{klo_completion_command[@]}" dev __complete --shell zsh --position "$CURRENT" -- "${zshWords}" 2>/dev/null)}`,
|
||||
].join('');
|
||||
const zshCompletionsCount = '$' + '{#completions[@]}';
|
||||
const zshCompletionCommand = '$' + '(eval "print -r -- $' + '{KLO_COMPLETION_COMMAND:-klo}")';
|
||||
|
||||
return [
|
||||
'#compdef klo',
|
||||
'',
|
||||
'_klo() {',
|
||||
' local -a completions',
|
||||
' local -a klo_completion_command',
|
||||
` klo_completion_command=("\${(@z)${zshCompletionCommand}}")`,
|
||||
` completions=("${zshCompletionCapture}")`,
|
||||
` if (( ${zshCompletionsCount} )); then`,
|
||||
" _describe 'klo completions' completions",
|
||||
' else',
|
||||
' _files',
|
||||
' fi',
|
||||
'}',
|
||||
'',
|
||||
'compdef _klo klo',
|
||||
'',
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
export async function installZshCompletion(): Promise<ZshCompletionInstallResult> {
|
||||
const homeDir = process.env.HOME || homedir();
|
||||
const zshConfigDir = process.env.ZDOTDIR || homeDir;
|
||||
const completionDir = join(homeDir, '.zfunc');
|
||||
const completionPath = join(completionDir, '_klo');
|
||||
const zshrcPath = join(zshConfigDir, '.zshrc');
|
||||
|
||||
await mkdir(completionDir, { recursive: true });
|
||||
await mkdir(dirname(zshrcPath), { recursive: true });
|
||||
await writeFile(completionPath, zshCompletionScript(), 'utf-8');
|
||||
|
||||
const existingZshrc = await readOptionalTextFile(zshrcPath);
|
||||
const nextZshrc = updateZshrcCompletionBlock(existingZshrc);
|
||||
await writeFile(zshrcPath, nextZshrc, 'utf-8');
|
||||
|
||||
return { completionPath, zshrcPath };
|
||||
}
|
||||
|
||||
export function completeCommanderInput(program: CommandUnknownOpts, request: CompletionRequest): string[] {
|
||||
const words = completionWordsForPosition(request.words, request.position);
|
||||
const tokens = stripProgramName(program, words);
|
||||
const current = tokens.at(-1) ?? '';
|
||||
const previous = tokens.slice(0, -1);
|
||||
const state = resolveCommandState(program, previous);
|
||||
|
||||
return candidatesForState(state, current).map(formatZshCandidate);
|
||||
}
|
||||
|
||||
function completionWordsForPosition(words: string[], position: number): string[] {
|
||||
if (!Number.isInteger(position) || position < 1) {
|
||||
return words;
|
||||
}
|
||||
return words.slice(0, position);
|
||||
}
|
||||
|
||||
function stripProgramName(program: CommandUnknownOpts, words: string[]): string[] {
|
||||
const [first, ...rest] = words;
|
||||
if (!first) {
|
||||
return [];
|
||||
}
|
||||
return first === program.name() || first.endsWith(`/${program.name()}`) ? rest : words;
|
||||
}
|
||||
|
||||
function resolveCommandState(program: CommandUnknownOpts, tokens: string[]): ResolveState {
|
||||
let command = program;
|
||||
let positionalIndex = 0;
|
||||
let pendingOption: Option | undefined;
|
||||
let positionalOnly = false;
|
||||
|
||||
for (let index = 0; index < tokens.length; index += 1) {
|
||||
const token = tokens[index];
|
||||
if (pendingOption) {
|
||||
pendingOption = undefined;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (token === '--') {
|
||||
positionalOnly = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!positionalOnly && token.startsWith('-')) {
|
||||
const option = findOption(command, optionNameFromToken(token));
|
||||
if (option && !token.includes('=') && optionTakesValue(option)) {
|
||||
if (index === tokens.length - 1) {
|
||||
pendingOption = option;
|
||||
} else if (option.required || !tokens[index + 1]?.startsWith('-')) {
|
||||
index += 1;
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
const child = findVisibleSubcommand(command, token);
|
||||
if (child) {
|
||||
command = child;
|
||||
positionalIndex = 0;
|
||||
continue;
|
||||
}
|
||||
|
||||
positionalIndex += 1;
|
||||
}
|
||||
|
||||
return { command, pendingOption, positionalIndex };
|
||||
}
|
||||
|
||||
function candidatesForState(state: ResolveState, current: string): CompletionCandidate[] {
|
||||
const optionValue = splitOptionValueToken(current);
|
||||
if (optionValue) {
|
||||
const option = findOption(state.command, optionValue.optionName);
|
||||
return choiceCandidates(option?.argChoices, optionValue.valuePrefix, optionValue.optionPrefix);
|
||||
}
|
||||
|
||||
if (state.pendingOption) {
|
||||
return choiceCandidates(state.pendingOption.argChoices, current);
|
||||
}
|
||||
|
||||
if (current.startsWith('-')) {
|
||||
return visibleOptions(state.command)
|
||||
.map(optionCandidate)
|
||||
.filter((candidate) => candidate.value.startsWith(current));
|
||||
}
|
||||
|
||||
const commandCandidates = visibleSubcommands(state.command)
|
||||
.map(commandCandidate)
|
||||
.filter((candidate) => candidate.value.startsWith(current));
|
||||
const argument = state.command.registeredArguments[state.positionalIndex];
|
||||
return [...commandCandidates, ...choiceCandidates(argument?.argChoices, current)];
|
||||
}
|
||||
|
||||
function visibleSubcommands(command: CommandUnknownOpts): CommandUnknownOpts[] {
|
||||
return command.commands.filter((subcommand) => (subcommand as CommandWithHiddenFlag)._hidden !== true);
|
||||
}
|
||||
|
||||
function findVisibleSubcommand(command: CommandUnknownOpts, name: string): CommandUnknownOpts | undefined {
|
||||
return visibleSubcommands(command).find(
|
||||
(subcommand) => subcommand.name() === name || subcommand.aliases().includes(name),
|
||||
);
|
||||
}
|
||||
|
||||
function visibleOptions(command: CommandUnknownOpts): Option[] {
|
||||
const options: Option[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const current of commandChain(command)) {
|
||||
for (const option of current.options) {
|
||||
if (option.hidden) {
|
||||
continue;
|
||||
}
|
||||
const key = option.long ?? option.short ?? option.flags;
|
||||
if (seen.has(key)) {
|
||||
continue;
|
||||
}
|
||||
seen.add(key);
|
||||
options.push(option);
|
||||
}
|
||||
}
|
||||
return options;
|
||||
}
|
||||
|
||||
function commandChain(command: CommandUnknownOpts): CommandUnknownOpts[] {
|
||||
const chain: CommandUnknownOpts[] = [];
|
||||
let current: CommandUnknownOpts | null = command;
|
||||
while (current) {
|
||||
chain.unshift(current);
|
||||
current = current.parent;
|
||||
}
|
||||
return chain;
|
||||
}
|
||||
|
||||
function findOption(command: CommandUnknownOpts, name: string): Option | undefined {
|
||||
return visibleOptions(command).find((option) => option.long === name || option.short === name);
|
||||
}
|
||||
|
||||
function optionTakesValue(option: Option): boolean {
|
||||
return option.required || option.optional;
|
||||
}
|
||||
|
||||
function optionNameFromToken(token: string): string {
|
||||
return token.split('=', 1)[0] ?? token;
|
||||
}
|
||||
|
||||
function splitOptionValueToken(
|
||||
token: string,
|
||||
): { optionName: string; optionPrefix: string; valuePrefix: string } | null {
|
||||
const separatorIndex = token.indexOf('=');
|
||||
if (!token.startsWith('-') || separatorIndex < 0) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
optionName: token.slice(0, separatorIndex),
|
||||
optionPrefix: token.slice(0, separatorIndex + 1),
|
||||
valuePrefix: token.slice(separatorIndex + 1),
|
||||
};
|
||||
}
|
||||
|
||||
function commandCandidate(command: CommandUnknownOpts): CompletionCandidate {
|
||||
return {
|
||||
value: command.name(),
|
||||
description: command.summary() || command.description(),
|
||||
};
|
||||
}
|
||||
|
||||
function optionCandidate(option: Option): CompletionCandidate {
|
||||
return {
|
||||
value: option.long ?? option.short ?? option.flags,
|
||||
description: option.description,
|
||||
};
|
||||
}
|
||||
|
||||
function choiceCandidates(
|
||||
choices: readonly string[] | undefined,
|
||||
prefix: string,
|
||||
completionPrefix = '',
|
||||
): CompletionCandidate[] {
|
||||
return (choices ?? [])
|
||||
.filter((choice) => choice.startsWith(prefix))
|
||||
.map((choice) => ({ value: `${completionPrefix}${choice}` }));
|
||||
}
|
||||
|
||||
function formatZshCandidate(candidate: CompletionCandidate): string {
|
||||
if (!candidate.description) {
|
||||
return escapeZshCompletion(candidate.value);
|
||||
}
|
||||
return `${escapeZshCompletion(candidate.value)}:${escapeZshDescription(candidate.description)}`;
|
||||
}
|
||||
|
||||
function escapeZshCompletion(value: string): string {
|
||||
return value.replace(/\\/g, '\\\\').replace(/:/g, '\\:');
|
||||
}
|
||||
|
||||
function escapeZshDescription(value: string): string {
|
||||
return value.replace(/\s+/g, ' ').replace(/\\/g, '\\\\').replace(/:/g, '\\:').trim();
|
||||
}
|
||||
|
||||
async function readOptionalTextFile(path: string): Promise<string> {
|
||||
try {
|
||||
return await readFile(path, 'utf-8');
|
||||
} catch (error) {
|
||||
if (isNodeError(error) && error.code === 'ENOENT') {
|
||||
return '';
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function updateZshrcCompletionBlock(contents: string): string {
|
||||
const withoutManagedBlock = contents.replace(KLO_COMPLETION_BLOCK_PATTERN, normalizeTrailingNewline);
|
||||
const hasCompinit = /^.*\bcompinit\b.*$/m.test(withoutManagedBlock);
|
||||
const block = zshrcCompletionBlock({ includeCompinit: !hasCompinit });
|
||||
|
||||
if (!hasCompinit) {
|
||||
return appendBlock(withoutManagedBlock, block);
|
||||
}
|
||||
|
||||
const compinitMatch = /^.*\bcompinit\b.*$/m.exec(withoutManagedBlock);
|
||||
if (!compinitMatch || compinitMatch.index === undefined) {
|
||||
return appendBlock(withoutManagedBlock, block);
|
||||
}
|
||||
|
||||
return [
|
||||
withoutManagedBlock.slice(0, compinitMatch.index),
|
||||
block,
|
||||
'\n',
|
||||
withoutManagedBlock.slice(compinitMatch.index),
|
||||
].join('');
|
||||
}
|
||||
|
||||
function zshrcCompletionBlock(options: { includeCompinit: boolean }): string {
|
||||
return [
|
||||
KLO_COMPLETION_BLOCK_START,
|
||||
'_klo_completion_command() {',
|
||||
' local dir="$PWD"',
|
||||
' while [[ "$dir" != "/" ]]; do',
|
||||
` if [[ -f "$dir/package.json" ]] && command grep -q '"name": "klo-workspace"' "$dir/package.json" 2>/dev/null; then`,
|
||||
' print -r -- "node $dir/scripts/run-klo.mjs --"',
|
||||
' return',
|
||||
' fi',
|
||||
' dir="' + '$' + '{dir:h}"',
|
||||
' done',
|
||||
' print -r -- "klo"',
|
||||
'}',
|
||||
"export KLO_COMPLETION_COMMAND='$(_klo_completion_command)'",
|
||||
'setopt complete_aliases',
|
||||
'fpath=("$HOME/.zfunc" $fpath)',
|
||||
...(options.includeCompinit ? ['autoload -Uz compinit', 'compinit'] : []),
|
||||
KLO_COMPLETION_BLOCK_END,
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
function appendBlock(contents: string, block: string): string {
|
||||
if (!contents.trim()) {
|
||||
return `${block}\n`;
|
||||
}
|
||||
return `${contents.replace(/\s*$/, '\n\n')}${block}\n`;
|
||||
}
|
||||
|
||||
function normalizeTrailingNewline(match: string): string {
|
||||
return match.startsWith('\n') || match.endsWith('\n') ? '\n' : '';
|
||||
}
|
||||
|
||||
function escapeRegExp(value: string): string {
|
||||
return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
}
|
||||
|
||||
function isNodeError(error: unknown): error is NodeJS.ErrnoException {
|
||||
return error instanceof Error && 'code' in error;
|
||||
}
|
||||
649
packages/cli/src/connection.test.ts
Normal file
649
packages/cli/src/connection.test.ts
Normal file
|
|
@ -0,0 +1,649 @@
|
|||
import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { initKloProject, parseKloProjectConfig } from '@klo/context/project';
|
||||
import type { KloConnectionDriver, KloScanConnector, KloSchemaSnapshot } from '@klo/context/scan';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { runKloConnection } from './connection.js';
|
||||
import { runKloCli, type KloCliIo } from './index.js';
|
||||
|
||||
function makeIo(options: { stdoutIsTty?: boolean; stdinIsTty?: boolean } = {}) {
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
return {
|
||||
io: {
|
||||
stdin: {
|
||||
isTTY: options.stdinIsTty,
|
||||
},
|
||||
stdout: {
|
||||
isTTY: options.stdoutIsTty,
|
||||
write: (chunk: string) => {
|
||||
stdout += chunk;
|
||||
},
|
||||
},
|
||||
stderr: {
|
||||
write: (chunk: string) => {
|
||||
stderr += chunk;
|
||||
},
|
||||
},
|
||||
},
|
||||
stdout: () => stdout,
|
||||
stderr: () => stderr,
|
||||
};
|
||||
}
|
||||
|
||||
function snapshotFor(driver: KloConnectionDriver, tableNames: string[]): KloSchemaSnapshot {
|
||||
return {
|
||||
connectionId: 'warehouse',
|
||||
driver,
|
||||
extractedAt: '2026-04-29T00:00:00.000Z',
|
||||
scope: {},
|
||||
metadata: {},
|
||||
tables: tableNames.map((name) => ({
|
||||
catalog: null,
|
||||
db: null,
|
||||
name,
|
||||
kind: 'table',
|
||||
comment: null,
|
||||
estimatedRows: null,
|
||||
columns: [],
|
||||
foreignKeys: [],
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
function nativeConnector(driver: KloConnectionDriver, tableNames: string[]) {
|
||||
const introspect = vi.fn(async () => snapshotFor(driver, tableNames));
|
||||
const cleanup = vi.fn(async () => undefined);
|
||||
const connector: KloScanConnector = {
|
||||
id: `${driver}:warehouse`,
|
||||
driver,
|
||||
capabilities: {
|
||||
structuralIntrospection: true,
|
||||
tableSampling: false,
|
||||
columnSampling: false,
|
||||
columnStats: false,
|
||||
readOnlySql: false,
|
||||
nestedAnalysis: false,
|
||||
eventStreamDiscovery: false,
|
||||
formalForeignKeys: false,
|
||||
estimatedRowCounts: false,
|
||||
},
|
||||
introspect,
|
||||
cleanup,
|
||||
};
|
||||
return { connector, introspect, cleanup };
|
||||
}
|
||||
|
||||
describe('runKloConnection', () => {
|
||||
let tempDir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'klo-cli-connection-'));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(tempDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('adds and lists env-referenced connections without resolving secrets', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await initKloProject({ projectDir, projectName: 'warehouse' });
|
||||
const io = makeIo();
|
||||
|
||||
await expect(
|
||||
runKloConnection(
|
||||
{
|
||||
command: 'add',
|
||||
projectDir,
|
||||
driver: 'postgres',
|
||||
connectionId: 'warehouse',
|
||||
url: 'env:DATABASE_URL',
|
||||
schemas: ['public'],
|
||||
readonly: true,
|
||||
force: false,
|
||||
allowLiteralCredentials: false,
|
||||
},
|
||||
io.io,
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(io.stdout()).toContain('Connection: warehouse');
|
||||
await expect(readFile(join(projectDir, 'klo.yaml'), 'utf-8')).resolves.toContain('url: env:DATABASE_URL');
|
||||
|
||||
const listIo = makeIo();
|
||||
await expect(runKloConnection({ command: 'list', projectDir }, listIo.io)).resolves.toBe(0);
|
||||
expect(listIo.stdout()).toContain('warehouse');
|
||||
expect(listIo.stdout()).toContain('postgres');
|
||||
});
|
||||
|
||||
it('removes a configured connection from klo.yaml without deleting local artifacts when forced', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await initKloProject({ projectDir, projectName: 'warehouse' });
|
||||
await runKloConnection(
|
||||
{
|
||||
command: 'add',
|
||||
projectDir,
|
||||
driver: 'sqlite',
|
||||
connectionId: 'warehouse',
|
||||
url: undefined,
|
||||
schemas: [],
|
||||
readonly: true,
|
||||
force: false,
|
||||
allowLiteralCredentials: false,
|
||||
},
|
||||
makeIo().io,
|
||||
);
|
||||
const artifactPath = join(projectDir, '.klo', 'artifacts', 'warehouse.txt');
|
||||
await mkdir(join(projectDir, '.klo', 'artifacts'), { recursive: true });
|
||||
await writeFile(artifactPath, 'keep me', 'utf-8');
|
||||
|
||||
const io = makeIo();
|
||||
|
||||
await expect(
|
||||
runKloConnection(
|
||||
{
|
||||
command: 'remove',
|
||||
projectDir,
|
||||
connectionId: 'warehouse',
|
||||
force: true,
|
||||
inputMode: 'disabled',
|
||||
},
|
||||
io.io,
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
const parsed = parseKloProjectConfig(await readFile(join(projectDir, 'klo.yaml'), 'utf-8'));
|
||||
expect(parsed.connections.warehouse).toBeUndefined();
|
||||
await expect(readFile(artifactPath, 'utf-8')).resolves.toBe('keep me');
|
||||
expect(io.stdout()).toContain('Connection removed from klo.yaml.');
|
||||
expect(io.stdout()).toContain(
|
||||
'Ingested artifacts from this connection remain in .klo/. Run klo dev artifacts to inspect.',
|
||||
);
|
||||
expect(io.stderr()).toBe('');
|
||||
});
|
||||
|
||||
it('requires --force when removing in non-interactive mode', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await initKloProject({ projectDir, projectName: 'warehouse' });
|
||||
await runKloConnection(
|
||||
{
|
||||
command: 'add',
|
||||
projectDir,
|
||||
driver: 'sqlite',
|
||||
connectionId: 'warehouse',
|
||||
url: undefined,
|
||||
schemas: [],
|
||||
readonly: true,
|
||||
force: false,
|
||||
allowLiteralCredentials: false,
|
||||
},
|
||||
makeIo().io,
|
||||
);
|
||||
const io = makeIo();
|
||||
|
||||
await expect(
|
||||
runKloConnection(
|
||||
{
|
||||
command: 'remove',
|
||||
projectDir,
|
||||
connectionId: 'warehouse',
|
||||
force: false,
|
||||
inputMode: 'disabled',
|
||||
},
|
||||
io.io,
|
||||
),
|
||||
).resolves.toBe(1);
|
||||
|
||||
expect(io.stderr()).toContain('connection remove warehouse requires --force when input is disabled or not interactive');
|
||||
});
|
||||
|
||||
it('returns a clear error when removing an unknown connection', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await initKloProject({ projectDir, projectName: 'warehouse' });
|
||||
const io = makeIo();
|
||||
|
||||
await expect(
|
||||
runKloConnection(
|
||||
{
|
||||
command: 'remove',
|
||||
projectDir,
|
||||
connectionId: 'missing',
|
||||
force: true,
|
||||
inputMode: 'disabled',
|
||||
},
|
||||
io.io,
|
||||
),
|
||||
).resolves.toBe(1);
|
||||
|
||||
expect(io.stderr()).toContain('Connection "missing" is not configured in klo.yaml');
|
||||
});
|
||||
|
||||
it('asks for confirmation before removing in an interactive terminal', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await initKloProject({ projectDir, projectName: 'warehouse' });
|
||||
await runKloConnection(
|
||||
{
|
||||
command: 'add',
|
||||
projectDir,
|
||||
driver: 'sqlite',
|
||||
connectionId: 'warehouse',
|
||||
url: undefined,
|
||||
schemas: [],
|
||||
readonly: true,
|
||||
force: false,
|
||||
allowLiteralCredentials: false,
|
||||
},
|
||||
makeIo().io,
|
||||
);
|
||||
const io = makeIo({ stdoutIsTty: true, stdinIsTty: true });
|
||||
const prompts = {
|
||||
confirm: vi.fn(async () => true),
|
||||
cancel: vi.fn(),
|
||||
};
|
||||
|
||||
await expect(
|
||||
runKloConnection(
|
||||
{
|
||||
command: 'remove',
|
||||
projectDir,
|
||||
connectionId: 'warehouse',
|
||||
force: false,
|
||||
},
|
||||
io.io,
|
||||
{ prompts },
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(prompts.confirm).toHaveBeenCalledWith({
|
||||
message: 'Remove connection "warehouse" from klo.yaml? Ingested artifacts will remain in .klo/.',
|
||||
initialValue: false,
|
||||
});
|
||||
});
|
||||
|
||||
it('runs public connect map as refresh, validate, and list over the low-level mapping runner', async () => {
|
||||
const io = makeIo();
|
||||
const runMapping = vi.fn(async (argv: string[], mappingIo: KloCliIo) => {
|
||||
if (argv[0] === 'refresh') {
|
||||
mappingIo.stdout.write('Discovery: 1 database\n');
|
||||
mappingIo.stdout.write('Unmapped discovered: 1\n');
|
||||
mappingIo.stdout.write('Stale mappings: 0\n');
|
||||
return 0;
|
||||
}
|
||||
if (argv[0] === 'validate') {
|
||||
mappingIo.stdout.write('Mapping validation passed: prod-metabase\n');
|
||||
return 0;
|
||||
}
|
||||
if (argv[0] === 'list') {
|
||||
mappingIo.stdout.write('1 -> [unmapped] (Analytics, sync: on, source: refresh)\n');
|
||||
return 0;
|
||||
}
|
||||
return 1;
|
||||
});
|
||||
|
||||
await expect(
|
||||
runKloConnection(
|
||||
{ command: 'map', projectDir: '/tmp/project', sourceConnectionId: 'prod-metabase', json: false },
|
||||
io.io,
|
||||
{ runMapping },
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(runMapping).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
['refresh', 'prod-metabase', '--auto-accept', '--project-dir', '/tmp/project'],
|
||||
expect.any(Object),
|
||||
);
|
||||
expect(runMapping).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
['validate', 'prod-metabase', '--project-dir', '/tmp/project'],
|
||||
expect.any(Object),
|
||||
);
|
||||
expect(runMapping).toHaveBeenNthCalledWith(
|
||||
3,
|
||||
['list', 'prod-metabase', '--project-dir', '/tmp/project'],
|
||||
expect.any(Object),
|
||||
);
|
||||
expect(io.stdout()).toContain('Mapping: prod-metabase');
|
||||
expect(io.stdout()).toContain('Discovery: 1 database');
|
||||
expect(io.stdout()).toContain('Mappings:');
|
||||
expect(io.stdout()).toContain('1 -> [unmapped]');
|
||||
expect(io.stdout()).toContain('Next:');
|
||||
expect(io.stdout()).toContain('klo ingest prod-metabase');
|
||||
expect(io.stdout()).toContain('klo dev mapping');
|
||||
expect(io.stderr()).toBe('');
|
||||
});
|
||||
|
||||
it('prints stable JSON for public connect map without leaking low-level stdout', async () => {
|
||||
const io = makeIo();
|
||||
const runMapping = vi.fn(async (argv: string[], mappingIo: KloCliIo) => {
|
||||
if (argv[0] === 'refresh') {
|
||||
mappingIo.stdout.write('Discovery: 1 connection\nUnmapped discovered: 0\nStale mappings: 0\n');
|
||||
return 0;
|
||||
}
|
||||
if (argv[0] === 'validate') {
|
||||
mappingIo.stdout.write('Mapping validation passed: prod-looker\n');
|
||||
return 0;
|
||||
}
|
||||
if (argv[0] === 'list') {
|
||||
expect(argv).toContain('--json');
|
||||
mappingIo.stdout.write(
|
||||
`${JSON.stringify(
|
||||
[
|
||||
{
|
||||
lookerConnectionName: 'analytics',
|
||||
kloConnectionId: 'prod-warehouse',
|
||||
source: 'klo.yaml',
|
||||
},
|
||||
],
|
||||
null,
|
||||
2,
|
||||
)}\n`,
|
||||
);
|
||||
return 0;
|
||||
}
|
||||
return 1;
|
||||
});
|
||||
|
||||
await expect(
|
||||
runKloConnection(
|
||||
{ command: 'map', projectDir: '/tmp/project', sourceConnectionId: 'prod-looker', json: true },
|
||||
io.io,
|
||||
{ runMapping },
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
const parsed = JSON.parse(io.stdout()) as {
|
||||
connectionId: string;
|
||||
refresh: { ok: boolean; output: string[] };
|
||||
validation: { ok: boolean; output: string[] };
|
||||
mappings: Array<{ lookerConnectionName: string; kloConnectionId: string; source: string }>;
|
||||
};
|
||||
expect(parsed).toEqual({
|
||||
connectionId: 'prod-looker',
|
||||
refresh: {
|
||||
ok: true,
|
||||
output: ['Discovery: 1 connection', 'Unmapped discovered: 0', 'Stale mappings: 0'],
|
||||
},
|
||||
validation: {
|
||||
ok: true,
|
||||
output: ['Mapping validation passed: prod-looker'],
|
||||
},
|
||||
mappings: [
|
||||
{
|
||||
lookerConnectionName: 'analytics',
|
||||
kloConnectionId: 'prod-warehouse',
|
||||
source: 'klo.yaml',
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(io.stderr()).toBe('');
|
||||
});
|
||||
|
||||
it('returns the refresh failure when public connect map cannot discover source metadata', async () => {
|
||||
const io = makeIo();
|
||||
const runMapping = vi.fn(async (argv: string[], mappingIo: KloCliIo) => {
|
||||
if (argv[0] === 'refresh') {
|
||||
mappingIo.stderr.write('Metabase API key is not configured\n');
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
});
|
||||
|
||||
await expect(
|
||||
runKloConnection(
|
||||
{ command: 'map', projectDir: '/tmp/project', sourceConnectionId: 'prod-metabase', json: false },
|
||||
io.io,
|
||||
{ runMapping },
|
||||
),
|
||||
).resolves.toBe(1);
|
||||
|
||||
expect(runMapping).toHaveBeenCalledTimes(1);
|
||||
expect(io.stdout()).toBe('');
|
||||
expect(io.stderr()).toContain('Metabase API key is not configured');
|
||||
});
|
||||
|
||||
it('rejects literal credential URLs unless explicitly allowed', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await initKloProject({ projectDir, projectName: 'warehouse' });
|
||||
const io = makeIo();
|
||||
|
||||
await expect(
|
||||
runKloConnection(
|
||||
{
|
||||
command: 'add',
|
||||
projectDir,
|
||||
driver: 'postgres',
|
||||
connectionId: 'warehouse',
|
||||
url: 'postgres://localhost:5432/warehouse',
|
||||
schemas: [],
|
||||
readonly: true,
|
||||
force: false,
|
||||
allowLiteralCredentials: false,
|
||||
},
|
||||
io.io,
|
||||
),
|
||||
).resolves.toBe(1);
|
||||
|
||||
expect(io.stderr()).toContain('Literal credential URLs require --allow-literal-credentials');
|
||||
});
|
||||
|
||||
it('warns before writing explicitly allowed literal credential URLs without echoing the URL', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await initKloProject({ projectDir, projectName: 'warehouse' });
|
||||
const io = makeIo();
|
||||
const literalUrl = 'postgres://localhost:5432/warehouse';
|
||||
|
||||
await expect(
|
||||
runKloConnection(
|
||||
{
|
||||
command: 'add',
|
||||
projectDir,
|
||||
driver: 'postgres',
|
||||
connectionId: 'warehouse',
|
||||
url: literalUrl,
|
||||
schemas: ['public'],
|
||||
readonly: true,
|
||||
force: false,
|
||||
allowLiteralCredentials: true,
|
||||
},
|
||||
io.io,
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(io.stderr()).toContain(
|
||||
'Warning: writing a literal credential URL to klo.yaml for connection "warehouse". Prefer env:NAME or file:/path references.',
|
||||
);
|
||||
expect(io.stderr()).not.toContain(literalUrl);
|
||||
await expect(readFile(join(projectDir, 'klo.yaml'), 'utf-8')).resolves.toContain(literalUrl);
|
||||
});
|
||||
|
||||
it('adds a Notion connection without writing token values', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await initKloProject({ projectDir, projectName: 'warehouse' });
|
||||
const io = makeIo();
|
||||
|
||||
await expect(
|
||||
runKloConnection(
|
||||
{
|
||||
command: 'add',
|
||||
projectDir,
|
||||
driver: 'notion',
|
||||
connectionId: 'notion-main',
|
||||
url: undefined,
|
||||
schemas: [],
|
||||
readonly: false,
|
||||
force: false,
|
||||
allowLiteralCredentials: false,
|
||||
notion: {
|
||||
authTokenRef: 'env:NOTION_AUTH_TOKEN',
|
||||
crawlMode: 'all_accessible',
|
||||
rootPageIds: [],
|
||||
rootDatabaseIds: [],
|
||||
rootDataSourceIds: [],
|
||||
maxPagesPerRun: 50,
|
||||
maxKnowledgeCreatesPerRun: 4,
|
||||
maxKnowledgeUpdatesPerRun: 12,
|
||||
},
|
||||
},
|
||||
io.io,
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
const yaml = await readFile(join(projectDir, 'klo.yaml'), 'utf-8');
|
||||
expect(yaml).toContain('driver: notion');
|
||||
expect(yaml).toContain('auth_token_ref: env:NOTION_AUTH_TOKEN');
|
||||
expect(yaml).toContain('crawl_mode: all_accessible');
|
||||
expect(yaml).toContain('max_pages_per_run: 50');
|
||||
expect(yaml).not.toContain('ntn_');
|
||||
expect(io.stdout()).toContain('Connection: notion-main');
|
||||
expect(io.stdout()).toContain('Driver: notion');
|
||||
});
|
||||
|
||||
it('runs connection notion pick --no-input through the public connection entrypoint', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await initKloProject({ projectDir, projectName: 'warehouse' });
|
||||
await runKloConnection(
|
||||
{
|
||||
command: 'add',
|
||||
projectDir,
|
||||
driver: 'notion',
|
||||
connectionId: 'notion-main',
|
||||
url: undefined,
|
||||
schemas: [],
|
||||
readonly: false,
|
||||
force: false,
|
||||
allowLiteralCredentials: false,
|
||||
notion: {
|
||||
authTokenRef: 'env:NOTION_AUTH_TOKEN',
|
||||
crawlMode: 'all_accessible',
|
||||
rootPageIds: [],
|
||||
rootDatabaseIds: ['database-1'],
|
||||
rootDataSourceIds: ['data-source-1'],
|
||||
maxPagesPerRun: 50,
|
||||
maxKnowledgeCreatesPerRun: 4,
|
||||
maxKnowledgeUpdatesPerRun: 12,
|
||||
},
|
||||
},
|
||||
makeIo().io,
|
||||
);
|
||||
const io = makeIo();
|
||||
|
||||
await expect(
|
||||
runKloCli(
|
||||
[
|
||||
'connection',
|
||||
'notion',
|
||||
'pick',
|
||||
'notion-main',
|
||||
'--project-dir',
|
||||
projectDir,
|
||||
'--no-input',
|
||||
'--root-page-id',
|
||||
'11111111222233334444555555555555',
|
||||
],
|
||||
io.io,
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
const yaml = await readFile(join(projectDir, 'klo.yaml'), 'utf-8');
|
||||
expect(yaml).toContain('crawl_mode: selected_roots');
|
||||
expect(yaml).toContain('11111111-2222-3333-4444-555555555555');
|
||||
expect(yaml).toContain('database-1');
|
||||
expect(yaml).toContain('data-source-1');
|
||||
expect(io.stdout()).toContain('Connection: notion-main');
|
||||
});
|
||||
|
||||
it('tests a configured connection through the native scan connector', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await initKloProject({ projectDir, projectName: 'warehouse' });
|
||||
await runKloConnection(
|
||||
{
|
||||
command: 'add',
|
||||
projectDir,
|
||||
driver: 'sqlite',
|
||||
connectionId: 'warehouse',
|
||||
url: undefined,
|
||||
schemas: [],
|
||||
readonly: true,
|
||||
force: false,
|
||||
allowLiteralCredentials: false,
|
||||
},
|
||||
makeIo().io,
|
||||
);
|
||||
const { connector, introspect, cleanup } = nativeConnector('sqlite', ['customers', 'orders']);
|
||||
const createScanConnector = vi.fn(async () => connector);
|
||||
const io = makeIo();
|
||||
|
||||
await expect(
|
||||
runKloConnection({ command: 'test', projectDir, connectionId: 'warehouse' }, io.io, {
|
||||
createScanConnector,
|
||||
}),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(createScanConnector).toHaveBeenCalledWith(expect.objectContaining({ projectDir }), 'warehouse');
|
||||
expect(introspect).toHaveBeenCalledWith(
|
||||
{
|
||||
connectionId: 'warehouse',
|
||||
driver: 'sqlite',
|
||||
mode: 'structural',
|
||||
dryRun: true,
|
||||
detectRelationships: false,
|
||||
},
|
||||
{ runId: 'connection-test-warehouse' },
|
||||
);
|
||||
expect(cleanup).toHaveBeenCalledTimes(1);
|
||||
expect(io.stdout()).toContain('Connection test passed: warehouse');
|
||||
expect(io.stdout()).toContain('Driver: sqlite');
|
||||
expect(io.stdout()).toContain('Tables: 2');
|
||||
});
|
||||
|
||||
it('cleans up the native scan connector when connection testing fails', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await initKloProject({ projectDir, projectName: 'warehouse' });
|
||||
await runKloConnection(
|
||||
{
|
||||
command: 'add',
|
||||
projectDir,
|
||||
driver: 'sqlite',
|
||||
connectionId: 'warehouse',
|
||||
url: undefined,
|
||||
schemas: [],
|
||||
readonly: true,
|
||||
force: false,
|
||||
allowLiteralCredentials: false,
|
||||
},
|
||||
makeIo().io,
|
||||
);
|
||||
const cleanup = vi.fn(async () => undefined);
|
||||
const connector: KloScanConnector = {
|
||||
id: 'sqlite:warehouse',
|
||||
driver: 'sqlite',
|
||||
capabilities: {
|
||||
structuralIntrospection: true,
|
||||
tableSampling: false,
|
||||
columnSampling: false,
|
||||
columnStats: false,
|
||||
readOnlySql: false,
|
||||
nestedAnalysis: false,
|
||||
eventStreamDiscovery: false,
|
||||
formalForeignKeys: false,
|
||||
estimatedRowCounts: false,
|
||||
},
|
||||
introspect: vi.fn(async () => {
|
||||
throw new Error('database file is unreadable');
|
||||
}),
|
||||
cleanup,
|
||||
};
|
||||
const io = makeIo();
|
||||
|
||||
await expect(
|
||||
runKloConnection({ command: 'test', projectDir, connectionId: 'warehouse' }, io.io, {
|
||||
createScanConnector: vi.fn(async () => connector),
|
||||
}),
|
||||
).resolves.toBe(1);
|
||||
|
||||
expect(cleanup).toHaveBeenCalledTimes(1);
|
||||
expect(io.stderr()).toContain('database file is unreadable');
|
||||
});
|
||||
});
|
||||
415
packages/cli/src/connection.ts
Normal file
415
packages/cli/src/connection.ts
Normal file
|
|
@ -0,0 +1,415 @@
|
|||
import { cancel, confirm, isCancel } from '@clack/prompts';
|
||||
import { type KloLocalProject, loadKloProject, serializeKloProjectConfig } from '@klo/context/project';
|
||||
import type { KloScanConnector } from '@klo/context/scan';
|
||||
import type { KloConnectionMappingArgs } from './commands/connection-mapping.js';
|
||||
import type { KloCliIo } from './index.js';
|
||||
import { createKloCliScanConnector } from './local-scan-connectors.js';
|
||||
import { profileMark } from './startup-profile.js';
|
||||
|
||||
profileMark('module:connection');
|
||||
|
||||
interface KloNotionConnectionCliConfig {
|
||||
authTokenRef: string;
|
||||
crawlMode: 'all_accessible' | 'selected_roots';
|
||||
rootPageIds: string[];
|
||||
rootDatabaseIds: string[];
|
||||
rootDataSourceIds: string[];
|
||||
maxPagesPerRun?: number;
|
||||
maxKnowledgeCreatesPerRun?: number;
|
||||
maxKnowledgeUpdatesPerRun?: number;
|
||||
}
|
||||
|
||||
type KloConnectionInputMode = 'disabled';
|
||||
|
||||
export type KloConnectionArgs =
|
||||
| { command: 'list'; projectDir: string }
|
||||
| {
|
||||
command: 'add';
|
||||
projectDir: string;
|
||||
driver: string;
|
||||
connectionId: string;
|
||||
url?: string;
|
||||
schemas: string[];
|
||||
readonly: boolean;
|
||||
force: boolean;
|
||||
allowLiteralCredentials: boolean;
|
||||
notion?: KloNotionConnectionCliConfig;
|
||||
}
|
||||
| { command: 'test'; projectDir: string; connectionId: string }
|
||||
| {
|
||||
command: 'remove';
|
||||
projectDir: string;
|
||||
connectionId: string;
|
||||
force: boolean;
|
||||
inputMode?: KloConnectionInputMode;
|
||||
}
|
||||
| {
|
||||
command: 'map';
|
||||
projectDir: string;
|
||||
sourceConnectionId: string;
|
||||
json: boolean;
|
||||
};
|
||||
|
||||
interface KloConnectionPromptAdapter {
|
||||
confirm(options: { message: string; initialValue?: boolean }): Promise<boolean>;
|
||||
cancel(message: string): void;
|
||||
}
|
||||
|
||||
interface KloConnectionIo extends KloCliIo {
|
||||
stdin?: { isTTY?: boolean };
|
||||
}
|
||||
|
||||
interface KloConnectionDeps {
|
||||
createScanConnector?: typeof createKloCliScanConnector;
|
||||
runMapping?: (argv: string[], io: KloCliIo) => Promise<number>;
|
||||
prompts?: KloConnectionPromptAdapter;
|
||||
}
|
||||
|
||||
function assertSafeConnectionId(connectionId: string): void {
|
||||
if (!/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(connectionId)) {
|
||||
throw new Error(`Unsafe connection id: ${connectionId}`);
|
||||
}
|
||||
}
|
||||
|
||||
function isCredentialReference(value: string): boolean {
|
||||
return value.startsWith('env:') || value.startsWith('file:');
|
||||
}
|
||||
|
||||
function literalCredentialWarning(connectionId: string): string {
|
||||
return `Warning: writing a literal credential URL to klo.yaml for connection "${connectionId}". Prefer env:NAME or file:/path references.`;
|
||||
}
|
||||
|
||||
function createClackConnectionPromptAdapter(): KloConnectionPromptAdapter {
|
||||
return {
|
||||
async confirm(options: { message: string; initialValue?: boolean }): Promise<boolean> {
|
||||
const value = await confirm(options);
|
||||
return isCancel(value) ? false : value;
|
||||
},
|
||||
cancel(message: string): void {
|
||||
cancel(message);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function isInteractiveConnectionIo(
|
||||
args: Extract<KloConnectionArgs, { command: 'remove' }>,
|
||||
io: KloConnectionIo,
|
||||
): boolean {
|
||||
return args.inputMode !== 'disabled' && io.stdin?.isTTY === true && io.stdout.isTTY === true;
|
||||
}
|
||||
|
||||
async function cleanupConnector(connector: KloScanConnector | null): Promise<void> {
|
||||
if (connector?.cleanup) {
|
||||
await connector.cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
async function testNativeConnection(
|
||||
project: KloLocalProject,
|
||||
connectionId: string,
|
||||
createScanConnector: typeof createKloCliScanConnector,
|
||||
): Promise<{ driver: string; tableCount: number }> {
|
||||
let connector: KloScanConnector | null = null;
|
||||
try {
|
||||
connector = await createScanConnector(project, connectionId);
|
||||
const snapshot = await connector.introspect(
|
||||
{
|
||||
connectionId,
|
||||
driver: connector.driver,
|
||||
mode: 'structural',
|
||||
dryRun: true,
|
||||
detectRelationships: false,
|
||||
},
|
||||
{ runId: `connection-test-${connectionId}` },
|
||||
);
|
||||
return {
|
||||
driver: connector.driver,
|
||||
tableCount: snapshot.tables.length,
|
||||
};
|
||||
} finally {
|
||||
await cleanupConnector(connector);
|
||||
}
|
||||
}
|
||||
|
||||
interface BufferedIo extends KloCliIo {
|
||||
stdoutText(): string;
|
||||
stderrText(): string;
|
||||
}
|
||||
|
||||
function createBufferedIo(): BufferedIo {
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
return {
|
||||
stdout: {
|
||||
write(chunk: string) {
|
||||
stdout += chunk;
|
||||
},
|
||||
},
|
||||
stderr: {
|
||||
write(chunk: string) {
|
||||
stderr += chunk;
|
||||
},
|
||||
},
|
||||
stdoutText() {
|
||||
return stdout;
|
||||
},
|
||||
stderrText() {
|
||||
return stderr;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function splitOutputLines(output: string): string[] {
|
||||
return output
|
||||
.split('\n')
|
||||
.map((line) => line.trim())
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
async function runLowLevelMapping(
|
||||
args: KloConnectionMappingArgs,
|
||||
argv: string[],
|
||||
io: KloCliIo,
|
||||
deps: KloConnectionDeps,
|
||||
): Promise<number> {
|
||||
if (deps.runMapping) {
|
||||
return await deps.runMapping(argv, io);
|
||||
}
|
||||
|
||||
const { runKloConnectionMapping } = await import('./commands/connection-mapping.js');
|
||||
return await runKloConnectionMapping(args, io);
|
||||
}
|
||||
|
||||
function parseMappingListJson(output: string): unknown[] {
|
||||
const trimmed = output.trim();
|
||||
if (!trimmed) {
|
||||
return [];
|
||||
}
|
||||
const parsed = JSON.parse(trimmed) as unknown;
|
||||
return Array.isArray(parsed) ? parsed : [];
|
||||
}
|
||||
|
||||
async function runPublicConnectionMap(
|
||||
args: Extract<KloConnectionArgs, { command: 'map' }>,
|
||||
io: KloCliIo,
|
||||
deps: KloConnectionDeps,
|
||||
): Promise<number> {
|
||||
const refreshIo = createBufferedIo();
|
||||
const refreshArgs: KloConnectionMappingArgs = {
|
||||
command: 'refresh',
|
||||
projectDir: args.projectDir,
|
||||
connectionId: args.sourceConnectionId,
|
||||
autoAccept: true,
|
||||
};
|
||||
const refreshCode = await runLowLevelMapping(
|
||||
refreshArgs,
|
||||
['refresh', args.sourceConnectionId, '--auto-accept', '--project-dir', args.projectDir],
|
||||
refreshIo,
|
||||
deps,
|
||||
);
|
||||
if (refreshCode !== 0) {
|
||||
io.stderr.write(
|
||||
refreshIo.stderrText() ||
|
||||
refreshIo.stdoutText() ||
|
||||
`Failed to refresh mapping metadata for ${args.sourceConnectionId}\n`,
|
||||
);
|
||||
return refreshCode;
|
||||
}
|
||||
|
||||
const validationIo = createBufferedIo();
|
||||
const validationArgs: KloConnectionMappingArgs = {
|
||||
command: 'validate',
|
||||
projectDir: args.projectDir,
|
||||
connectionId: args.sourceConnectionId,
|
||||
};
|
||||
const validationCode = await runLowLevelMapping(
|
||||
validationArgs,
|
||||
['validate', args.sourceConnectionId, '--project-dir', args.projectDir],
|
||||
validationIo,
|
||||
deps,
|
||||
);
|
||||
if (validationCode !== 0) {
|
||||
io.stderr.write(
|
||||
validationIo.stderrText() || validationIo.stdoutText() || `Mapping validation failed for ${args.sourceConnectionId}\n`,
|
||||
);
|
||||
return validationCode;
|
||||
}
|
||||
|
||||
const listIo = createBufferedIo();
|
||||
const listArgv = ['list', args.sourceConnectionId, '--project-dir', args.projectDir];
|
||||
const listArgs: KloConnectionMappingArgs = {
|
||||
command: 'list',
|
||||
projectDir: args.projectDir,
|
||||
connectionId: args.sourceConnectionId,
|
||||
json: args.json,
|
||||
};
|
||||
const listCode = await runLowLevelMapping(listArgs, args.json ? [...listArgv, '--json'] : listArgv, listIo, deps);
|
||||
if (listCode !== 0) {
|
||||
io.stderr.write(listIo.stderrText() || listIo.stdoutText() || `Failed to list mappings for ${args.sourceConnectionId}\n`);
|
||||
return listCode;
|
||||
}
|
||||
|
||||
if (args.json) {
|
||||
io.stdout.write(
|
||||
`${JSON.stringify(
|
||||
{
|
||||
connectionId: args.sourceConnectionId,
|
||||
refresh: { ok: true, output: splitOutputLines(refreshIo.stdoutText()) },
|
||||
validation: { ok: true, output: splitOutputLines(validationIo.stdoutText()) },
|
||||
mappings: parseMappingListJson(listIo.stdoutText()),
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}\n`,
|
||||
);
|
||||
return 0;
|
||||
}
|
||||
|
||||
io.stdout.write(`Mapping: ${args.sourceConnectionId}\n`);
|
||||
io.stdout.write(refreshIo.stdoutText());
|
||||
io.stdout.write(validationIo.stdoutText());
|
||||
io.stdout.write('\nMappings:\n');
|
||||
io.stdout.write(listIo.stdoutText().trim() ? listIo.stdoutText() : 'No mappings found.\n');
|
||||
io.stdout.write('\nNext:\n');
|
||||
io.stdout.write(` klo ingest ${args.sourceConnectionId}\n`);
|
||||
io.stdout.write(` klo dev mapping list ${args.sourceConnectionId}\n`);
|
||||
return 0;
|
||||
}
|
||||
|
||||
export async function runKloConnection(
|
||||
args: KloConnectionArgs,
|
||||
io: KloConnectionIo = process,
|
||||
deps: KloConnectionDeps = {},
|
||||
): Promise<number> {
|
||||
try {
|
||||
if (args.command === 'map') {
|
||||
return await runPublicConnectionMap(args, io, deps);
|
||||
}
|
||||
|
||||
const project = await loadKloProject({ projectDir: args.projectDir });
|
||||
if (args.command === 'list') {
|
||||
const entries = Object.entries(project.config.connections).sort(([a], [b]) => a.localeCompare(b));
|
||||
if (entries.length === 0) {
|
||||
io.stdout.write('No connections configured. Run `klo connection add <id> --driver <driver>` to add one.\n');
|
||||
return 0;
|
||||
}
|
||||
const idWidth = Math.max('ID'.length, ...entries.map(([id]) => id.length));
|
||||
const driverWidth = Math.max(
|
||||
'DRIVER'.length,
|
||||
...entries.map(([, c]) => (c.driver ?? 'unknown').length),
|
||||
);
|
||||
io.stdout.write(`${'ID'.padEnd(idWidth)} ${'DRIVER'.padEnd(driverWidth)}\n`);
|
||||
for (const [id, connection] of entries) {
|
||||
io.stdout.write(`${id.padEnd(idWidth)} ${(connection.driver ?? 'unknown').padEnd(driverWidth)}\n`);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (args.command === 'add') {
|
||||
assertSafeConnectionId(args.connectionId);
|
||||
const hasLiteralCredentialUrl = !!args.url && !isCredentialReference(args.url);
|
||||
if (hasLiteralCredentialUrl && !args.allowLiteralCredentials) {
|
||||
throw new Error('Literal credential URLs require --allow-literal-credentials');
|
||||
}
|
||||
if (hasLiteralCredentialUrl) {
|
||||
io.stderr.write(`${literalCredentialWarning(args.connectionId)}\n`);
|
||||
}
|
||||
if (project.config.connections[args.connectionId] && !args.force) {
|
||||
throw new Error(`Connection "${args.connectionId}" already exists; pass --force to replace it`);
|
||||
}
|
||||
const connectionConfig =
|
||||
args.driver === 'notion' && args.notion
|
||||
? {
|
||||
driver: 'notion',
|
||||
auth_token_ref: args.notion.authTokenRef,
|
||||
crawl_mode: args.notion.crawlMode,
|
||||
root_page_ids: args.notion.rootPageIds,
|
||||
root_database_ids: args.notion.rootDatabaseIds,
|
||||
root_data_source_ids: args.notion.rootDataSourceIds,
|
||||
...(args.notion.maxPagesPerRun !== undefined ? { max_pages_per_run: args.notion.maxPagesPerRun } : {}),
|
||||
...(args.notion.maxKnowledgeCreatesPerRun !== undefined
|
||||
? { max_knowledge_creates_per_run: args.notion.maxKnowledgeCreatesPerRun }
|
||||
: {}),
|
||||
...(args.notion.maxKnowledgeUpdatesPerRun !== undefined
|
||||
? { max_knowledge_updates_per_run: args.notion.maxKnowledgeUpdatesPerRun }
|
||||
: {}),
|
||||
}
|
||||
: {
|
||||
driver: args.driver,
|
||||
...(args.url ? { url: args.url } : {}),
|
||||
...(args.schemas.length > 0 ? { schemas: args.schemas } : {}),
|
||||
readonly: args.readonly,
|
||||
};
|
||||
const nextConfig = {
|
||||
...project.config,
|
||||
connections: {
|
||||
...project.config.connections,
|
||||
[args.connectionId]: connectionConfig,
|
||||
},
|
||||
};
|
||||
await project.fileStore.writeFile(
|
||||
'klo.yaml',
|
||||
serializeKloProjectConfig(nextConfig),
|
||||
'klo',
|
||||
'klo@example.com',
|
||||
`Update KLO connection: ${args.connectionId}`,
|
||||
);
|
||||
io.stdout.write(`Connection: ${args.connectionId}\n`);
|
||||
io.stdout.write(`Driver: ${args.driver}\n`);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (args.command === 'remove') {
|
||||
if (!project.config.connections[args.connectionId]) {
|
||||
throw new Error(`Connection "${args.connectionId}" is not configured in klo.yaml`);
|
||||
}
|
||||
|
||||
if (!args.force) {
|
||||
if (!isInteractiveConnectionIo(args, io)) {
|
||||
throw new Error(
|
||||
`connection remove ${args.connectionId} requires --force when input is disabled or not interactive`,
|
||||
);
|
||||
}
|
||||
|
||||
const prompts = deps.prompts ?? createClackConnectionPromptAdapter();
|
||||
const confirmed = await prompts.confirm({
|
||||
message: `Remove connection "${args.connectionId}" from klo.yaml? Ingested artifacts will remain in .klo/.`,
|
||||
initialValue: false,
|
||||
});
|
||||
if (!confirmed) {
|
||||
prompts.cancel('Connection removal cancelled.');
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
const { [args.connectionId]: _removedConnection, ...connections } = project.config.connections;
|
||||
const nextConfig = {
|
||||
...project.config,
|
||||
connections,
|
||||
};
|
||||
await project.fileStore.writeFile(
|
||||
'klo.yaml',
|
||||
serializeKloProjectConfig(nextConfig),
|
||||
'klo',
|
||||
'klo@example.com',
|
||||
`Remove KLO connection: ${args.connectionId}`,
|
||||
);
|
||||
io.stdout.write('Connection removed from klo.yaml.\n');
|
||||
io.stdout.write('Ingested artifacts from this connection remain in .klo/. Run klo dev artifacts to inspect.\n');
|
||||
return 0;
|
||||
}
|
||||
|
||||
const result = await testNativeConnection(
|
||||
project,
|
||||
args.connectionId,
|
||||
deps.createScanConnector ?? createKloCliScanConnector,
|
||||
);
|
||||
io.stdout.write(`Connection test passed: ${args.connectionId}\n`);
|
||||
io.stdout.write(`Driver: ${result.driver}\n`);
|
||||
io.stdout.write(`Tables: ${result.tableCount}\n`);
|
||||
return 0;
|
||||
} catch (error) {
|
||||
io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
303
packages/cli/src/context-build-view.test.ts
Normal file
303
packages/cli/src/context-build-view.test.ts
Normal file
|
|
@ -0,0 +1,303 @@
|
|||
import { buildDefaultKloProjectConfig, type KloProjectConfig } from '@klo/context/project';
|
||||
import { describe, expect, it, vi } from 'vitest';
|
||||
import type { KloPublicIngestProject, KloPublicIngestTargetResult } from './public-ingest.js';
|
||||
import {
|
||||
extractProgressMessage,
|
||||
initViewState,
|
||||
parseIngestSummary,
|
||||
parseScanSummary,
|
||||
renderContextBuildView,
|
||||
runContextBuild,
|
||||
} from './context-build-view.js';
|
||||
|
||||
function makeIo(options: { isTTY?: boolean } = {}) {
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
return {
|
||||
io: {
|
||||
stdout: {
|
||||
isTTY: options.isTTY,
|
||||
write: (chunk: string) => {
|
||||
stdout += chunk;
|
||||
},
|
||||
},
|
||||
stderr: {
|
||||
write: (chunk: string) => {
|
||||
stderr += chunk;
|
||||
},
|
||||
},
|
||||
},
|
||||
stdout: () => stdout,
|
||||
stderr: () => stderr,
|
||||
};
|
||||
}
|
||||
|
||||
function projectWithConnections(connections: KloProjectConfig['connections']): KloPublicIngestProject {
|
||||
return {
|
||||
projectDir: '/tmp/project',
|
||||
config: {
|
||||
...buildDefaultKloProjectConfig('warehouse'),
|
||||
connections,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function successResult(connectionId: string, driver: string, operation: 'scan' | 'source-ingest'): KloPublicIngestTargetResult {
|
||||
return {
|
||||
connectionId,
|
||||
driver,
|
||||
steps: [
|
||||
{ operation: 'scan', status: operation === 'scan' ? 'done' : 'skipped' },
|
||||
{ operation: 'source-ingest', status: operation === 'source-ingest' ? 'done' : 'skipped' },
|
||||
{ operation: 'enrich', status: 'skipped' },
|
||||
{ operation: 'memory-update', status: operation === 'source-ingest' ? 'done' : 'skipped' },
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
function failedResult(connectionId: string, driver: string, operation: 'scan' | 'source-ingest'): KloPublicIngestTargetResult {
|
||||
return {
|
||||
connectionId,
|
||||
driver,
|
||||
steps: [
|
||||
{ operation: 'scan', status: operation === 'scan' ? 'failed' : 'skipped', detail: `${connectionId} failed at scan.` },
|
||||
{ operation: 'source-ingest', status: operation === 'source-ingest' ? 'failed' : 'skipped' },
|
||||
{ operation: 'enrich', status: 'skipped' },
|
||||
{ operation: 'memory-update', status: 'not-run' },
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
describe('extractProgressMessage', () => {
|
||||
it('extracts percentage and message from scan progress', () => {
|
||||
expect(extractProgressMessage('\r[45%] Scanning tables...[K')).toBe('[45%] Scanning tables...');
|
||||
});
|
||||
|
||||
it('extracts from permanent progress lines', () => {
|
||||
expect(extractProgressMessage('[100%] Done\n')).toBe('[100%] Done');
|
||||
});
|
||||
|
||||
it('returns null for non-progress output', () => {
|
||||
expect(extractProgressMessage('KLO scan completed\n')).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('parseScanSummary', () => {
|
||||
it('extracts table count from scan output', () => {
|
||||
expect(parseScanSummary('Semantic layer comparison found 5 changes across 42 tables')).toBe('42 tables');
|
||||
});
|
||||
|
||||
it('handles singular form', () => {
|
||||
expect(parseScanSummary('found 1 change across 1 table')).toBe('1 tables');
|
||||
});
|
||||
|
||||
it('returns null when no match', () => {
|
||||
expect(parseScanSummary('No changes detected')).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('parseIngestSummary', () => {
|
||||
it('extracts work units and saved memory', () => {
|
||||
expect(parseIngestSummary('Work units: 5\nSaved memory: 3 wiki, 2 SL')).toBe('5 work units · 3 wiki, 2 SL');
|
||||
});
|
||||
|
||||
it('extracts work units alone when no saved memory', () => {
|
||||
expect(parseIngestSummary('Work units: 5\nStatus: done')).toBe('5 work units');
|
||||
});
|
||||
|
||||
it('extracts saved memory alone when no work units', () => {
|
||||
expect(parseIngestSummary('Saved memory: 3 wiki, 2 SL')).toBe('3 wiki, 2 SL');
|
||||
});
|
||||
|
||||
it('returns null when no match', () => {
|
||||
expect(parseIngestSummary('Status: done')).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('initViewState', () => {
|
||||
it('partitions targets into primary and context sources', () => {
|
||||
const state = initViewState([
|
||||
{ connectionId: 'warehouse', driver: 'postgres', operation: 'scan', debugCommand: '', steps: ['scan'] },
|
||||
{ connectionId: 'dbt-main', driver: 'dbt', operation: 'source-ingest', adapter: 'dbt', debugCommand: '', steps: ['source-ingest', 'memory-update'] },
|
||||
]);
|
||||
|
||||
expect(state.primarySources).toHaveLength(1);
|
||||
expect(state.primarySources[0].target.connectionId).toBe('warehouse');
|
||||
expect(state.contextSources).toHaveLength(1);
|
||||
expect(state.contextSources[0].target.connectionId).toBe('dbt-main');
|
||||
expect(state.frame).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('renderContextBuildView', () => {
|
||||
it('renders all-queued state', () => {
|
||||
const state = initViewState([
|
||||
{ connectionId: 'warehouse', driver: 'postgres', operation: 'scan', debugCommand: '', steps: ['scan'] },
|
||||
{ connectionId: 'dbt-main', driver: 'dbt', operation: 'source-ingest', adapter: 'dbt', debugCommand: '', steps: ['source-ingest', 'memory-update'] },
|
||||
]);
|
||||
|
||||
const output = renderContextBuildView(state, { styled: false });
|
||||
expect(output).toContain('Building KLO context');
|
||||
expect(output).toContain('Primary sources:');
|
||||
expect(output).toContain('warehouse');
|
||||
expect(output).toContain('queued');
|
||||
expect(output).toContain('Context sources:');
|
||||
expect(output).toContain('dbt-main');
|
||||
});
|
||||
|
||||
it('renders completed state with summary', () => {
|
||||
const state = initViewState([
|
||||
{ connectionId: 'warehouse', driver: 'postgres', operation: 'scan', debugCommand: '', steps: ['scan'] },
|
||||
]);
|
||||
state.primarySources[0].status = 'done';
|
||||
state.primarySources[0].elapsedMs = 72000;
|
||||
state.primarySources[0].summaryText = '42 tables';
|
||||
|
||||
const output = renderContextBuildView(state, { styled: false });
|
||||
expect(output).toContain('42 tables');
|
||||
expect(output).toContain('1m12s');
|
||||
});
|
||||
|
||||
it('renders failed state', () => {
|
||||
const state = initViewState([
|
||||
{ connectionId: 'warehouse', driver: 'postgres', operation: 'scan', debugCommand: '', steps: ['scan'] },
|
||||
]);
|
||||
state.primarySources[0].status = 'failed';
|
||||
|
||||
const output = renderContextBuildView(state, { styled: false });
|
||||
expect(output).toContain('✗');
|
||||
expect(output).toContain('failed');
|
||||
});
|
||||
|
||||
it('omits empty groups', () => {
|
||||
const state = initViewState([
|
||||
{ connectionId: 'dbt-main', driver: 'dbt', operation: 'source-ingest', adapter: 'dbt', debugCommand: '', steps: ['source-ingest', 'memory-update'] },
|
||||
]);
|
||||
|
||||
const output = renderContextBuildView(state, { styled: false });
|
||||
expect(output).not.toContain('Primary sources:');
|
||||
expect(output).toContain('Context sources:');
|
||||
});
|
||||
});
|
||||
|
||||
describe('runContextBuild', () => {
|
||||
it('executes scan targets before source-ingest targets', async () => {
|
||||
const io = makeIo();
|
||||
const project = projectWithConnections({
|
||||
dbt_main: { driver: 'dbt' },
|
||||
warehouse: { driver: 'postgres' },
|
||||
});
|
||||
const callOrder: string[] = [];
|
||||
const executeTarget = vi.fn(async (target) => {
|
||||
callOrder.push(target.connectionId);
|
||||
return successResult(target.connectionId, target.driver, target.operation);
|
||||
});
|
||||
|
||||
const result = await runContextBuild(
|
||||
project,
|
||||
{ projectDir: '/tmp/project', inputMode: 'disabled' },
|
||||
io.io,
|
||||
{ executeTarget, now: () => 1000 },
|
||||
);
|
||||
|
||||
expect(result).toEqual({ exitCode: 0, detached: false });
|
||||
expect(callOrder).toEqual(['warehouse', 'dbt_main']);
|
||||
});
|
||||
|
||||
it('returns exit code 1 when any target fails', async () => {
|
||||
const io = makeIo();
|
||||
const project = projectWithConnections({
|
||||
warehouse: { driver: 'postgres' },
|
||||
});
|
||||
const executeTarget = vi.fn(async (target) => failedResult(target.connectionId, target.driver, target.operation));
|
||||
|
||||
const result = await runContextBuild(
|
||||
project,
|
||||
{ projectDir: '/tmp/project', inputMode: 'disabled' },
|
||||
io.io,
|
||||
{ executeTarget, now: () => 1000 },
|
||||
);
|
||||
|
||||
expect(result).toEqual({ exitCode: 1, detached: false });
|
||||
});
|
||||
|
||||
it('renders final view for non-TTY output', async () => {
|
||||
const io = makeIo();
|
||||
const project = projectWithConnections({
|
||||
warehouse: { driver: 'postgres' },
|
||||
dbt_main: { driver: 'dbt' },
|
||||
});
|
||||
const executeTarget = vi.fn(async (target) => successResult(target.connectionId, target.driver, target.operation));
|
||||
|
||||
await runContextBuild(
|
||||
project,
|
||||
{ projectDir: '/tmp/project', inputMode: 'disabled' },
|
||||
io.io,
|
||||
{ executeTarget, now: () => 1000 },
|
||||
);
|
||||
|
||||
const output = io.stdout();
|
||||
expect(output).toContain('Building KLO context');
|
||||
expect(output).toContain('Primary sources:');
|
||||
expect(output).toContain('warehouse');
|
||||
expect(output).toContain('Context sources:');
|
||||
expect(output).toContain('dbt_main');
|
||||
});
|
||||
|
||||
it('passes scan mode and detect relationships through to target execution', async () => {
|
||||
const io = makeIo();
|
||||
const project = projectWithConnections({ warehouse: { driver: 'postgres' } });
|
||||
const executeTarget = vi.fn(async (target) => successResult(target.connectionId, target.driver, target.operation));
|
||||
|
||||
await runContextBuild(
|
||||
project,
|
||||
{ projectDir: '/tmp/project', inputMode: 'disabled', scanMode: 'enriched', detectRelationships: true },
|
||||
io.io,
|
||||
{ executeTarget, now: () => 1000 },
|
||||
);
|
||||
|
||||
expect(executeTarget).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ connectionId: 'warehouse', operation: 'scan' }),
|
||||
expect.objectContaining({ scanMode: 'enriched', detectRelationships: true }),
|
||||
expect.anything(),
|
||||
{},
|
||||
);
|
||||
});
|
||||
|
||||
it('exits immediately with paused message when d is pressed', async () => {
|
||||
const mockExit = vi.spyOn(process, 'exit').mockImplementation(() => {
|
||||
throw new Error('process.exit');
|
||||
});
|
||||
const io = makeIo();
|
||||
const project = projectWithConnections({
|
||||
warehouse: { driver: 'postgres' },
|
||||
dbt_main: { driver: 'dbt' },
|
||||
});
|
||||
let triggerDetach: (() => void) | null = null;
|
||||
const executeTarget = vi.fn(async (target) => {
|
||||
if (target.connectionId === 'warehouse') triggerDetach?.();
|
||||
return successResult(target.connectionId, target.driver, target.operation);
|
||||
});
|
||||
|
||||
await expect(
|
||||
runContextBuild(
|
||||
project,
|
||||
{ projectDir: '/tmp/project', inputMode: 'disabled' },
|
||||
io.io,
|
||||
{
|
||||
executeTarget,
|
||||
now: () => 1000,
|
||||
setupKeystroke: (onDetach) => {
|
||||
triggerDetach = onDetach;
|
||||
return () => {};
|
||||
},
|
||||
},
|
||||
),
|
||||
).rejects.toThrow('process.exit');
|
||||
|
||||
expect(mockExit).toHaveBeenCalledWith(0);
|
||||
expect(io.stdout()).toContain('Context build continuing in the background.');
|
||||
expect(io.stdout()).toContain('Resume: klo setup --project-dir /tmp/project');
|
||||
mockExit.mockRestore();
|
||||
});
|
||||
});
|
||||
414
packages/cli/src/context-build-view.ts
Normal file
414
packages/cli/src/context-build-view.ts
Normal file
|
|
@ -0,0 +1,414 @@
|
|||
import { spawn } from 'node:child_process';
|
||||
import { mkdirSync, openSync } from 'node:fs';
|
||||
import { join, resolve } from 'node:path';
|
||||
import type { KloCliIo } from './index.js';
|
||||
import type {
|
||||
KloPublicIngestArgs,
|
||||
KloPublicIngestPlanTarget,
|
||||
KloPublicIngestProject,
|
||||
KloPublicIngestTargetResult,
|
||||
} from './public-ingest.js';
|
||||
import { buildPublicIngestPlan, executePublicIngestTarget } from './public-ingest.js';
|
||||
import { formatDuration } from './demo-metrics.js';
|
||||
import { profileMark } from './startup-profile.js';
|
||||
|
||||
profileMark('module:context-build-view');
|
||||
|
||||
const SPINNER_FRAMES = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'] as const;
|
||||
const ESC = String.fromCharCode(0x1b);
|
||||
|
||||
export interface ContextBuildTargetState {
|
||||
target: KloPublicIngestPlanTarget;
|
||||
status: 'queued' | 'running' | 'done' | 'failed';
|
||||
detailLine: string | null;
|
||||
summaryText: string | null;
|
||||
startedAt: number | null;
|
||||
elapsedMs: number;
|
||||
}
|
||||
|
||||
export interface ContextBuildViewState {
|
||||
primarySources: ContextBuildTargetState[];
|
||||
contextSources: ContextBuildTargetState[];
|
||||
frame: number;
|
||||
}
|
||||
|
||||
export interface ContextBuildArgs {
|
||||
projectDir: string;
|
||||
inputMode: 'auto' | 'disabled';
|
||||
scanMode?: 'structural' | 'enriched';
|
||||
detectRelationships?: boolean;
|
||||
}
|
||||
|
||||
export interface ContextBuildResult {
|
||||
exitCode: number;
|
||||
detached: boolean;
|
||||
}
|
||||
|
||||
export interface ContextBuildDeps {
|
||||
executeTarget?: typeof executePublicIngestTarget;
|
||||
now?: () => number;
|
||||
setupKeystroke?: (onDetach: () => void, onCtrlC: () => void) => (() => void) | null;
|
||||
onDetach?: () => void;
|
||||
}
|
||||
|
||||
// --- Rendering ---
|
||||
|
||||
function green(text: string): string {
|
||||
return `${ESC}[32m${text}${ESC}[39m`;
|
||||
}
|
||||
|
||||
function red(text: string): string {
|
||||
return `${ESC}[31m${text}${ESC}[39m`;
|
||||
}
|
||||
|
||||
function cyan(text: string): string {
|
||||
return `${ESC}[36m${text}${ESC}[39m`;
|
||||
}
|
||||
|
||||
function dim(text: string): string {
|
||||
return `${ESC}[2m${text}${ESC}[22m`;
|
||||
}
|
||||
|
||||
function statusIcon(status: ContextBuildTargetState['status'], frame: number, styled: boolean): string {
|
||||
if (!styled) {
|
||||
switch (status) {
|
||||
case 'done':
|
||||
return '✓';
|
||||
case 'failed':
|
||||
return '✗';
|
||||
case 'running':
|
||||
return SPINNER_FRAMES[frame % SPINNER_FRAMES.length] ?? '⠋';
|
||||
default:
|
||||
return '·';
|
||||
}
|
||||
}
|
||||
switch (status) {
|
||||
case 'done':
|
||||
return green('✓');
|
||||
case 'failed':
|
||||
return red('✗');
|
||||
case 'running':
|
||||
return cyan(SPINNER_FRAMES[frame % SPINNER_FRAMES.length] ?? '⠋');
|
||||
default:
|
||||
return dim('·');
|
||||
}
|
||||
}
|
||||
|
||||
function targetDetail(target: ContextBuildTargetState, styled: boolean): string {
|
||||
if (target.status === 'done') {
|
||||
const parts: string[] = [];
|
||||
if (target.summaryText) parts.push(target.summaryText);
|
||||
parts.push(formatDuration(target.elapsedMs));
|
||||
return parts.join(' · ');
|
||||
}
|
||||
if (target.status === 'failed') {
|
||||
return styled ? red('failed') : 'failed';
|
||||
}
|
||||
if (target.status === 'running') {
|
||||
return target.detailLine ?? (target.target.operation === 'scan' ? 'scanning...' : 'ingesting...');
|
||||
}
|
||||
return styled ? dim('queued') : 'queued';
|
||||
}
|
||||
|
||||
function columnWidth(state: ContextBuildViewState): number {
|
||||
const all = [...state.primarySources, ...state.contextSources];
|
||||
return Math.max(12, ...all.map((t) => t.target.connectionId.length)) + 2;
|
||||
}
|
||||
|
||||
function renderTargetLine(target: ContextBuildTargetState, frame: number, styled: boolean, width: number): string {
|
||||
return ` ${statusIcon(target.status, frame, styled)} ${target.target.connectionId.padEnd(width)} ${targetDetail(target, styled)}`;
|
||||
}
|
||||
|
||||
function renderTargetGroup(
|
||||
label: string,
|
||||
targets: ContextBuildTargetState[],
|
||||
frame: number,
|
||||
styled: boolean,
|
||||
width: number,
|
||||
): string[] {
|
||||
if (targets.length === 0) return [];
|
||||
return ['', ` ${label}:`, ...targets.map((t) => renderTargetLine(t, frame, styled, width))];
|
||||
}
|
||||
|
||||
function resumeCommand(projectDir?: string): string {
|
||||
return projectDir ? `klo setup --project-dir ${projectDir}` : 'klo setup';
|
||||
}
|
||||
|
||||
export function renderContextBuildView(
|
||||
state: ContextBuildViewState,
|
||||
options: { styled?: boolean; showHint?: boolean; projectDir?: string } = {},
|
||||
): string {
|
||||
const styled = options.styled ?? true;
|
||||
const width = columnWidth(state);
|
||||
const lines: string[] = [
|
||||
'',
|
||||
'Building KLO context',
|
||||
'─────────────────────',
|
||||
...renderTargetGroup('Primary sources', state.primarySources, state.frame, styled, width),
|
||||
...renderTargetGroup('Context sources', state.contextSources, state.frame, styled, width),
|
||||
'',
|
||||
];
|
||||
const hasActive = [...state.primarySources, ...state.contextSources].some(
|
||||
(t) => t.status === 'running' || t.status === 'queued',
|
||||
);
|
||||
if (options.showHint && hasActive) {
|
||||
const hint = ` d to detach · ${resumeCommand(options.projectDir)} to resume`;
|
||||
lines.push(styled ? dim(hint) : hint);
|
||||
lines.push('');
|
||||
}
|
||||
return `${lines.join('\n')}\n`;
|
||||
}
|
||||
|
||||
// --- IO Capture ---
|
||||
|
||||
const ESC_K_RE = new RegExp(`${ESC.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\[K`, 'g');
|
||||
|
||||
export function extractProgressMessage(chunk: string): string | null {
|
||||
const cleaned = chunk.replace(/^\r/, '').replace(ESC_K_RE, '').replace(/\n$/, '').trim();
|
||||
const match = cleaned.match(/^\[(\d+)%\]\s*(.+)$/);
|
||||
return match ? `[${match[1]}%] ${match[2]}` : null;
|
||||
}
|
||||
|
||||
export function parseScanSummary(output: string): string | null {
|
||||
const match = output.match(/(\d+) changes? across (\d+) tables?/);
|
||||
return match ? `${match[2]} tables` : null;
|
||||
}
|
||||
|
||||
export function parseIngestSummary(output: string): string | null {
|
||||
const parts: string[] = [];
|
||||
const workUnits = output.match(/Work units: (\d+)/);
|
||||
if (workUnits) parts.push(`${workUnits[1]} work units`);
|
||||
const savedMemory = output.match(/Saved memory: (.+)/);
|
||||
if (savedMemory) parts.push(savedMemory[1]);
|
||||
return parts.length > 0 ? parts.join(' · ') : null;
|
||||
}
|
||||
|
||||
interface CapturedIo {
|
||||
io: KloCliIo;
|
||||
captured(): string;
|
||||
}
|
||||
|
||||
function createCaptureIo(onProgress: (message: string) => void, isTTY: boolean): CapturedIo {
|
||||
let buffer = '';
|
||||
return {
|
||||
io: {
|
||||
stdout: {
|
||||
isTTY,
|
||||
write(chunk: string) {
|
||||
buffer += chunk;
|
||||
const progress = extractProgressMessage(chunk);
|
||||
if (progress) onProgress(progress);
|
||||
},
|
||||
},
|
||||
stderr: {
|
||||
write(chunk: string) {
|
||||
buffer += chunk;
|
||||
},
|
||||
},
|
||||
},
|
||||
captured: () => buffer,
|
||||
};
|
||||
}
|
||||
|
||||
// --- Repaint ---
|
||||
|
||||
function createRepainter(io: KloCliIo) {
|
||||
let lastLineCount = 0;
|
||||
|
||||
return {
|
||||
paint(content: string) {
|
||||
if (lastLineCount > 0) {
|
||||
io.stdout.write(`${ESC}[${lastLineCount}A\r`);
|
||||
}
|
||||
io.stdout.write(content);
|
||||
io.stdout.write(`${ESC}[J`);
|
||||
lastLineCount = (content.match(/\n/g) ?? []).length;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// --- Background build ---
|
||||
|
||||
function resolveKloEntryScript(): string | null {
|
||||
const argv1 = process.argv[1];
|
||||
if (argv1 && (argv1.endsWith('.js') || argv1.endsWith('.ts') || argv1.endsWith('.mjs'))) {
|
||||
return argv1;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function spawnBackgroundBuild(projectDir: string): { logPath: string } | null {
|
||||
const entryScript = resolveKloEntryScript();
|
||||
if (!entryScript) return null;
|
||||
|
||||
const resolvedDir = resolve(projectDir);
|
||||
const logDir = join(resolvedDir, '.klo', 'setup');
|
||||
mkdirSync(logDir, { recursive: true });
|
||||
const logPath = join(logDir, 'context-build.log');
|
||||
const logFd = openSync(logPath, 'w');
|
||||
|
||||
const child = spawn(
|
||||
process.execPath,
|
||||
[entryScript, 'setup', 'context', 'build', '--project-dir', resolvedDir, '--no-input'],
|
||||
{ detached: true, stdio: ['ignore', logFd, logFd] },
|
||||
);
|
||||
child.unref();
|
||||
return { logPath };
|
||||
}
|
||||
|
||||
// --- Keystroke handling ---
|
||||
|
||||
function defaultSetupKeystroke(onDetach: () => void, onCtrlC: () => void): (() => void) | null {
|
||||
const stdin = process.stdin;
|
||||
if (!stdin.isTTY || typeof stdin.setRawMode !== 'function') {
|
||||
return null;
|
||||
}
|
||||
stdin.setRawMode(true);
|
||||
stdin.resume();
|
||||
const onData = (data: Buffer) => {
|
||||
const char = data.toString();
|
||||
if (char === 'd' || char === 'D') onDetach();
|
||||
else if (char === '\x03') onCtrlC();
|
||||
};
|
||||
stdin.on('data', onData);
|
||||
return () => {
|
||||
stdin.off('data', onData);
|
||||
if (typeof stdin.setRawMode === 'function') stdin.setRawMode(false);
|
||||
stdin.pause();
|
||||
};
|
||||
}
|
||||
|
||||
// --- Orchestration ---
|
||||
|
||||
function makeTargetState(target: KloPublicIngestPlanTarget): ContextBuildTargetState {
|
||||
return { target, status: 'queued', detailLine: null, summaryText: null, startedAt: null, elapsedMs: 0 };
|
||||
}
|
||||
|
||||
export function initViewState(targets: KloPublicIngestPlanTarget[]): ContextBuildViewState {
|
||||
return {
|
||||
primarySources: targets.filter((t) => t.operation === 'scan').map(makeTargetState),
|
||||
contextSources: targets.filter((t) => t.operation === 'source-ingest').map(makeTargetState),
|
||||
frame: 0,
|
||||
};
|
||||
}
|
||||
|
||||
export async function runContextBuild(
|
||||
project: KloPublicIngestProject,
|
||||
args: ContextBuildArgs,
|
||||
io: KloCliIo,
|
||||
deps: ContextBuildDeps = {},
|
||||
): Promise<ContextBuildResult> {
|
||||
const plan = buildPublicIngestPlan(project, { projectDir: args.projectDir, all: true });
|
||||
const state = initViewState(plan.targets);
|
||||
const isTTY = io.stdout.isTTY === true;
|
||||
const nowFn = deps.now ?? (() => Date.now());
|
||||
|
||||
const repainter = isTTY ? createRepainter(io) : null;
|
||||
const viewOpts = { styled: true, projectDir: args.projectDir };
|
||||
const paint = (hint: boolean) => repainter?.paint(renderContextBuildView(state, { ...viewOpts, showHint: hint }));
|
||||
paint(true);
|
||||
|
||||
let spinnerInterval: ReturnType<typeof setInterval> | null = null;
|
||||
if (repainter) {
|
||||
spinnerInterval = setInterval(() => {
|
||||
state.frame++;
|
||||
for (const t of [...state.primarySources, ...state.contextSources]) {
|
||||
if (t.status === 'running' && t.startedAt !== null) {
|
||||
t.elapsedMs = nowFn() - t.startedAt;
|
||||
}
|
||||
}
|
||||
paint(true);
|
||||
}, 140);
|
||||
}
|
||||
|
||||
const orderedTargets = [...state.primarySources, ...state.contextSources];
|
||||
const execTarget = deps.executeTarget ?? executePublicIngestTarget;
|
||||
|
||||
let detached = false;
|
||||
let cleanupKeystroke: (() => void) | null = null;
|
||||
|
||||
if (isTTY || deps.setupKeystroke) {
|
||||
const cleanup = () => {
|
||||
if (spinnerInterval) clearInterval(spinnerInterval);
|
||||
cleanupKeystroke?.();
|
||||
};
|
||||
cleanupKeystroke = (deps.setupKeystroke ?? defaultSetupKeystroke)(
|
||||
() => {
|
||||
cleanup();
|
||||
deps.onDetach?.();
|
||||
const bg = spawnBackgroundBuild(args.projectDir);
|
||||
io.stdout.write('\n\nContext build continuing in the background.\n');
|
||||
if (bg) io.stdout.write(`Log: ${bg.logPath}\n`);
|
||||
io.stdout.write(`Status: klo setup context status --project-dir ${resolve(args.projectDir)}\n`);
|
||||
io.stdout.write(`Resume: ${resumeCommand(args.projectDir)}\n`);
|
||||
process.exit(0);
|
||||
},
|
||||
() => {
|
||||
cleanup();
|
||||
io.stdout.write('\n\nContext build stopped. Nothing is running in the background.\n');
|
||||
io.stdout.write(`Resume: ${resumeCommand(args.projectDir)}\n`);
|
||||
process.exit(130);
|
||||
},
|
||||
);
|
||||
}
|
||||
const runArgs: Extract<KloPublicIngestArgs, { command: 'run' }> = {
|
||||
command: 'run',
|
||||
projectDir: args.projectDir,
|
||||
all: true,
|
||||
json: false,
|
||||
inputMode: args.inputMode,
|
||||
scanMode: args.scanMode,
|
||||
detectRelationships: args.detectRelationships,
|
||||
};
|
||||
|
||||
let hasFailure = false;
|
||||
|
||||
try {
|
||||
for (const targetState of orderedTargets) {
|
||||
if (detached) break;
|
||||
|
||||
targetState.status = 'running';
|
||||
targetState.startedAt = nowFn();
|
||||
paint(true);
|
||||
|
||||
const capture = createCaptureIo(
|
||||
(message) => {
|
||||
targetState.detailLine = message;
|
||||
paint(true);
|
||||
},
|
||||
false,
|
||||
);
|
||||
|
||||
const result = await execTarget(targetState.target, runArgs, capture.io, {});
|
||||
|
||||
targetState.elapsedMs = nowFn() - (targetState.startedAt ?? nowFn());
|
||||
const failed = result.steps.some((s) => s.status === 'failed');
|
||||
targetState.status = failed ? 'failed' : 'done';
|
||||
targetState.detailLine = null;
|
||||
if (!failed) {
|
||||
targetState.summaryText =
|
||||
targetState.target.operation === 'scan'
|
||||
? parseScanSummary(capture.captured())
|
||||
: parseIngestSummary(capture.captured());
|
||||
}
|
||||
if (failed) hasFailure = true;
|
||||
|
||||
paint(true);
|
||||
}
|
||||
} finally {
|
||||
if (spinnerInterval) clearInterval(spinnerInterval);
|
||||
cleanupKeystroke?.();
|
||||
}
|
||||
|
||||
if (detached) {
|
||||
return { exitCode: 0, detached: true };
|
||||
}
|
||||
|
||||
if (!repainter) {
|
||||
io.stdout.write(renderContextBuildView(state, { styled: false }));
|
||||
} else {
|
||||
paint(false);
|
||||
}
|
||||
|
||||
return { exitCode: hasFailure ? 1 : 0, detached: false };
|
||||
}
|
||||
272
packages/cli/src/demo-assets.test.ts
Normal file
272
packages/cli/src/demo-assets.test.ts
Normal file
|
|
@ -0,0 +1,272 @@
|
|||
import { access, readFile, rm, stat, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { afterEach, describe, expect, it } from 'vitest';
|
||||
import {
|
||||
DEMO_ADAPTER,
|
||||
DEMO_CONNECTION_ID,
|
||||
DEMO_FULL_JOB_ID,
|
||||
DEMO_REPLAY_FILE,
|
||||
defaultDemoProjectDir,
|
||||
ensureDemoProject,
|
||||
inspectDemoProjectState,
|
||||
loadPackagedDemoReplay,
|
||||
loadProjectDemoReplay,
|
||||
resetDemoProject,
|
||||
} from './demo-assets.js';
|
||||
import { writeDemoReplay } from './demo-replay-store.js';
|
||||
|
||||
const packagedDemoSource = 'packaged-orbit-demo';
|
||||
|
||||
function packagedDemoAssetPath(relativePath: string): string {
|
||||
return fileURLToPath(new URL(`../assets/demo/orbit/${relativePath}`, import.meta.url));
|
||||
}
|
||||
|
||||
async function readPackagedJson<T>(relativePath: string): Promise<T> {
|
||||
return JSON.parse(await readFile(packagedDemoAssetPath(relativePath), 'utf-8')) as T;
|
||||
}
|
||||
|
||||
describe('demo assets', () => {
|
||||
const projectDir = join(tmpdir(), `klo-demo-assets-${process.pid}`);
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(projectDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('resolves the default demo root under the OS temp directory', () => {
|
||||
const dir = defaultDemoProjectDir();
|
||||
expect(dir.startsWith(join(tmpdir(), 'klo-demo-'))).toBe(true);
|
||||
expect(dir).toMatch(/klo-demo-[a-f0-9]{8}$/);
|
||||
});
|
||||
|
||||
it('exports the packaged Orbit demo identity', () => {
|
||||
expect(DEMO_CONNECTION_ID).toBe('orbit_demo');
|
||||
expect(DEMO_ADAPTER).toBe('live-database');
|
||||
expect(DEMO_REPLAY_FILE).toBe('replay.memory-flow.v1.json');
|
||||
expect(DEMO_FULL_JOB_ID).toBe('demo-full-ingest');
|
||||
});
|
||||
|
||||
it('ships the seeded demo bundle required by the May 6 PRD', async () => {
|
||||
const manifest = await readPackagedJson<{
|
||||
demoAssetSchemaVersion: number;
|
||||
mode: string;
|
||||
source: string;
|
||||
sources: {
|
||||
warehouse: { tables: number; rowCounts: Record<string, number> };
|
||||
dbt: { models: number; sourceTables: number };
|
||||
bi: { explores: number; dashboards: number };
|
||||
notion: { pages: number };
|
||||
};
|
||||
name: string;
|
||||
displayName: string;
|
||||
generated: {
|
||||
semanticLayer: { path: string; sourceCount: number };
|
||||
knowledge: { pageCount: number };
|
||||
links: { linkCount: number };
|
||||
};
|
||||
}>('manifest.json');
|
||||
|
||||
expect(manifest).toMatchObject({
|
||||
demoAssetSchemaVersion: 2,
|
||||
name: 'orbit',
|
||||
displayName: 'Orbit Demo',
|
||||
mode: 'seeded',
|
||||
source: packagedDemoSource,
|
||||
});
|
||||
expect(manifest.sources.warehouse.tables).toBeGreaterThanOrEqual(5);
|
||||
expect(manifest.sources.warehouse.tables).toBeLessThanOrEqual(10);
|
||||
expect(Object.keys(manifest.sources.warehouse.rowCounts).sort()).toEqual([
|
||||
'accounts',
|
||||
'arr_movements',
|
||||
'contracts',
|
||||
'invoices',
|
||||
'plans',
|
||||
'purchase_requests',
|
||||
'support_tickets',
|
||||
'users',
|
||||
]);
|
||||
expect(manifest.sources.dbt.models).toBeGreaterThanOrEqual(3);
|
||||
expect(manifest.sources.dbt.models).toBeLessThanOrEqual(6);
|
||||
expect(manifest.sources.bi.explores).toBeGreaterThanOrEqual(2);
|
||||
expect(manifest.sources.bi.dashboards).toBeGreaterThanOrEqual(2);
|
||||
expect(manifest.sources.notion.pages).toBeGreaterThanOrEqual(5);
|
||||
expect(manifest.generated.semanticLayer.sourceCount).toBeGreaterThanOrEqual(5);
|
||||
expect(manifest.generated.knowledge.pageCount).toBeGreaterThanOrEqual(10);
|
||||
expect(manifest.generated.links.linkCount).toBeGreaterThanOrEqual(10);
|
||||
|
||||
const dbStat = await stat(packagedDemoAssetPath('demo.db'));
|
||||
expect(dbStat.size).toBeGreaterThan(0);
|
||||
expect(dbStat.size).toBeLessThan(10 * 1024 * 1024);
|
||||
|
||||
await expect(access(packagedDemoAssetPath('raw-sources/warehouse/accounts.csv'))).resolves.toBeUndefined();
|
||||
await expect(access(packagedDemoAssetPath('raw-sources/dbt/schema.yml'))).resolves.toBeUndefined();
|
||||
await expect(access(packagedDemoAssetPath('raw-sources/bi/revenue_exec.dashboard.lookml'))).resolves.toBeUndefined();
|
||||
await expect(access(packagedDemoAssetPath('raw-sources/notion/revenue-reporting-policy.md'))).resolves.toBeUndefined();
|
||||
expect(manifest.generated.semanticLayer.path).toBe('semantic-layer/orbit_demo');
|
||||
|
||||
await expect(access(packagedDemoAssetPath('semantic-layer/orbit_demo/accounts.yaml'))).resolves.toBeUndefined();
|
||||
await expect(access(packagedDemoAssetPath('knowledge/global/arr-contract-first.md'))).resolves.toBeUndefined();
|
||||
await expect(access(packagedDemoAssetPath('links/provenance.json'))).resolves.toBeUndefined();
|
||||
await expect(access(packagedDemoAssetPath('reports/seeded-demo-report.json'))).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it('initializes a flat demo project without writing literal credentials', async () => {
|
||||
const result = await ensureDemoProject({ projectDir, force: false });
|
||||
|
||||
expect(result.projectDir).toBe(projectDir);
|
||||
await expect(access(join(projectDir, 'demo.db'))).resolves.toBeUndefined();
|
||||
await expect(access(join(projectDir, 'state.sqlite'))).resolves.toBeUndefined();
|
||||
await expect(access(join(projectDir, 'reports'))).resolves.toBeUndefined();
|
||||
await expect(access(join(projectDir, 'semantic-layer'))).resolves.toBeUndefined();
|
||||
await expect(access(join(projectDir, 'knowledge'))).resolves.toBeUndefined();
|
||||
await expect(access(join(projectDir, 'replays', 'replay.memory-flow.v1.json'))).resolves.toBeUndefined();
|
||||
await expect(access(join(projectDir, 'raw-sources'))).resolves.toBeUndefined();
|
||||
await expect(access(join(projectDir, '_schema'))).rejects.toMatchObject({ code: 'ENOENT' });
|
||||
|
||||
const config = await readFile(join(projectDir, 'klo.yaml'), 'utf-8');
|
||||
expect(config).toContain('backend: anthropic');
|
||||
expect(config).toContain('api_key: env:ANTHROPIC_API_KEY');
|
||||
expect(config).not.toContain('sk-ant-');
|
||||
});
|
||||
|
||||
it('rejects an existing demo project unless force is set', async () => {
|
||||
await ensureDemoProject({ projectDir, force: false });
|
||||
await expect(ensureDemoProject({ projectDir, force: false })).rejects.toThrow('Demo project already exists');
|
||||
await expect(ensureDemoProject({ projectDir, force: true })).resolves.toMatchObject({ projectDir });
|
||||
});
|
||||
|
||||
it('loads packaged and copied demo replays', async () => {
|
||||
const packaged = await loadPackagedDemoReplay();
|
||||
expect(packaged.runId).toBe('demo-seeded-orbit');
|
||||
expect(packaged.connectionId).toBe('orbit_demo');
|
||||
expect(packaged.metadata?.mode).toBe('seeded');
|
||||
|
||||
await ensureDemoProject({ projectDir, force: false });
|
||||
const copied = await loadProjectDemoReplay(projectDir);
|
||||
expect(copied).toEqual(packaged);
|
||||
});
|
||||
|
||||
it('loads the latest local replay before the packaged replay', async () => {
|
||||
await ensureDemoProject({ projectDir, force: false });
|
||||
await writeDemoReplay(
|
||||
projectDir,
|
||||
{
|
||||
metadata: {
|
||||
schemaVersion: 1,
|
||||
mode: 'full',
|
||||
origin: 'captured',
|
||||
timing: 'captured',
|
||||
capturedAt: '2026-05-01T10:00:03.000Z',
|
||||
sourceReportId: null,
|
||||
sourceReportPath: 'raw-sources/orbit_demo/live-database/sync/scan-report.json',
|
||||
fallbackReason: null,
|
||||
},
|
||||
runId: 'demo-full-run',
|
||||
connectionId: 'orbit_demo',
|
||||
adapter: 'live-database',
|
||||
status: 'done',
|
||||
sourceDir: null,
|
||||
syncId: 'sync',
|
||||
reportPath: 'raw-sources/orbit_demo/live-database/sync/scan-report.json',
|
||||
errors: [],
|
||||
events: [{ type: 'report_created', runId: 'scan-run' }],
|
||||
plannedWorkUnits: [],
|
||||
details: { actions: [], provenance: [], transcripts: [] },
|
||||
},
|
||||
{ label: 'full' },
|
||||
);
|
||||
|
||||
await expect(loadProjectDemoReplay(projectDir)).resolves.toMatchObject({
|
||||
runId: 'demo-full-run',
|
||||
metadata: { mode: 'full', origin: 'captured' },
|
||||
});
|
||||
});
|
||||
|
||||
it('reports missing, ready, and corrupted demo project state', async () => {
|
||||
await expect(inspectDemoProjectState(projectDir)).resolves.toEqual({
|
||||
status: 'missing',
|
||||
projectDir,
|
||||
missing: ['klo.yaml', 'demo.db', 'state.sqlite', 'replays/replay.memory-flow.v1.json'],
|
||||
});
|
||||
|
||||
await ensureDemoProject({ projectDir, force: false });
|
||||
await expect(inspectDemoProjectState(projectDir)).resolves.toEqual({
|
||||
status: 'ready',
|
||||
projectDir,
|
||||
missing: [],
|
||||
});
|
||||
|
||||
await rm(join(projectDir, 'demo.db'), { force: true });
|
||||
await expect(inspectDemoProjectState(projectDir)).resolves.toEqual({
|
||||
status: 'corrupt',
|
||||
projectDir,
|
||||
missing: ['demo.db'],
|
||||
});
|
||||
});
|
||||
|
||||
it('requires explicit force for demo reset and recreates packaged assets', async () => {
|
||||
await ensureDemoProject({ projectDir, force: false });
|
||||
await rm(join(projectDir, 'demo.db'), { force: true });
|
||||
|
||||
await expect(resetDemoProject({ projectDir, force: false })).rejects.toThrow(
|
||||
`klo setup demo reset is destructive; pass --force to recreate ${projectDir}`,
|
||||
);
|
||||
|
||||
await expect(resetDemoProject({ projectDir, force: true })).resolves.toMatchObject({ projectDir });
|
||||
await expect(access(join(projectDir, 'demo.db'))).resolves.toBeUndefined();
|
||||
await expect(inspectDemoProjectState(projectDir)).resolves.toMatchObject({ status: 'ready' });
|
||||
});
|
||||
|
||||
it('preserves a user-edited klo.yaml across reset --force', async () => {
|
||||
await ensureDemoProject({ projectDir, force: false });
|
||||
const customConfig = [
|
||||
'project: klo-demo-orbit',
|
||||
'connections:',
|
||||
` ${DEMO_CONNECTION_ID}:`,
|
||||
' driver: sqlite',
|
||||
` path: ${JSON.stringify(join(projectDir, 'demo.db'))}`,
|
||||
' readonly: true',
|
||||
'storage:',
|
||||
' state: sqlite',
|
||||
' search: sqlite-fts5',
|
||||
' git:',
|
||||
' auto_commit: true',
|
||||
' author: klo <klo@example.com>',
|
||||
'llm:',
|
||||
' provider:',
|
||||
' backend: vertex',
|
||||
' vertex:',
|
||||
' project: example-gcp-project',
|
||||
' location: us-east5',
|
||||
' models:',
|
||||
' default: claude-sonnet-4-6',
|
||||
'ingest:',
|
||||
' adapters:',
|
||||
` - ${DEMO_ADAPTER}`,
|
||||
' embeddings:',
|
||||
' backend: none',
|
||||
' dimensions: 8',
|
||||
' workUnits:',
|
||||
' stepBudget: 40',
|
||||
' maxConcurrency: 1',
|
||||
' failureMode: continue',
|
||||
'',
|
||||
].join('\n');
|
||||
await writeFile(join(projectDir, 'klo.yaml'), customConfig, 'utf-8');
|
||||
|
||||
await resetDemoProject({ projectDir, force: true });
|
||||
|
||||
const preserved = await readFile(join(projectDir, 'klo.yaml'), 'utf-8');
|
||||
expect(preserved).toBe(customConfig);
|
||||
expect(preserved).toContain('backend: vertex');
|
||||
expect(preserved).not.toContain('backend: anthropic');
|
||||
await expect(inspectDemoProjectState(projectDir)).resolves.toMatchObject({ status: 'ready' });
|
||||
});
|
||||
|
||||
it('still writes the default klo.yaml on reset when none exists', async () => {
|
||||
await resetDemoProject({ projectDir, force: true });
|
||||
const config = await readFile(join(projectDir, 'klo.yaml'), 'utf-8');
|
||||
expect(config).toContain('backend: anthropic');
|
||||
});
|
||||
});
|
||||
281
packages/cli/src/demo-assets.ts
Normal file
281
packages/cli/src/demo-assets.ts
Normal file
|
|
@ -0,0 +1,281 @@
|
|||
import { constants as fsConstants } from 'node:fs';
|
||||
import { access, copyFile, cp, mkdir, readFile, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { dirname, join, resolve } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { randomBytes } from 'node:crypto';
|
||||
import type { MemoryFlowReplayInput } from '@klo/context/ingest/memory-flow';
|
||||
import { loadDemoReplayFile, loadLatestDemoReplay } from './demo-replay-store.js';
|
||||
|
||||
interface DemoProjectResult {
|
||||
projectDir: string;
|
||||
configPath: string;
|
||||
databasePath: string;
|
||||
replayPath: string;
|
||||
}
|
||||
|
||||
interface EnsureDemoProjectOptions {
|
||||
projectDir: string;
|
||||
force: boolean;
|
||||
}
|
||||
|
||||
type DemoProjectStateStatus = 'missing' | 'ready' | 'corrupt';
|
||||
|
||||
interface DemoProjectState {
|
||||
status: DemoProjectStateStatus;
|
||||
projectDir: string;
|
||||
missing: string[];
|
||||
}
|
||||
|
||||
export const DEMO_CONNECTION_ID = 'orbit_demo';
|
||||
export const DEMO_ADAPTER = 'live-database';
|
||||
export const DEMO_REPLAY_FILE = 'replay.memory-flow.v1.json';
|
||||
export const DEMO_FULL_JOB_ID = 'demo-full-ingest';
|
||||
|
||||
const REQUIRED_BASE_PROJECT_PATHS = [
|
||||
'klo.yaml',
|
||||
'demo.db',
|
||||
'state.sqlite',
|
||||
join('replays', DEMO_REPLAY_FILE),
|
||||
] as const;
|
||||
|
||||
const REQUIRED_PACKAGED_BASE_ASSET_PATHS = ['demo.db', 'manifest.json', DEMO_REPLAY_FILE] as const;
|
||||
|
||||
const REQUIRED_SEEDED_ASSET_PATHS = [
|
||||
'demo.db',
|
||||
'manifest.json',
|
||||
DEMO_REPLAY_FILE,
|
||||
join('raw-sources', 'warehouse', 'accounts.csv'),
|
||||
join('raw-sources', 'dbt', 'schema.yml'),
|
||||
join('raw-sources', 'bi', 'revenue_exec.dashboard.lookml'),
|
||||
join('raw-sources', 'notion', 'revenue-reporting-policy.md'),
|
||||
join('semantic-layer', 'orbit_demo', 'accounts.yaml'),
|
||||
join('knowledge', 'global', 'arr-contract-first.md'),
|
||||
join('links', 'provenance.json'),
|
||||
join('reports', 'seeded-demo-report.json'),
|
||||
] as const;
|
||||
|
||||
function assetDir(): string {
|
||||
return fileURLToPath(new URL('../assets/demo/orbit/', import.meta.url));
|
||||
}
|
||||
|
||||
async function exists(path: string): Promise<boolean> {
|
||||
try {
|
||||
await access(path, fsConstants.F_OK);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export function defaultDemoProjectDir(): string {
|
||||
const suffix = randomBytes(4).toString('hex');
|
||||
return join(tmpdir(), `klo-demo-${suffix}`);
|
||||
}
|
||||
|
||||
export async function inspectDemoProjectState(projectDir: string): Promise<DemoProjectState> {
|
||||
const root = resolve(projectDir);
|
||||
const missing: string[] = [];
|
||||
|
||||
for (const relativePath of REQUIRED_BASE_PROJECT_PATHS) {
|
||||
if (!(await exists(join(root, relativePath)))) {
|
||||
missing.push(relativePath);
|
||||
}
|
||||
}
|
||||
|
||||
if (missing.length === REQUIRED_BASE_PROJECT_PATHS.length) {
|
||||
return { status: 'missing', projectDir: root, missing };
|
||||
}
|
||||
|
||||
if (missing.length > 0) {
|
||||
return { status: 'corrupt', projectDir: root, missing };
|
||||
}
|
||||
|
||||
return { status: 'ready', projectDir: root, missing: [] };
|
||||
}
|
||||
|
||||
export async function resetDemoProject(options: EnsureDemoProjectOptions): Promise<DemoProjectResult> {
|
||||
const projectDir = resolve(options.projectDir);
|
||||
if (!options.force) {
|
||||
throw new Error(`klo setup demo reset is destructive; pass --force to recreate ${projectDir}`);
|
||||
}
|
||||
|
||||
const preservedConfig = await readExistingConfig(join(projectDir, 'klo.yaml'));
|
||||
const result = await ensureDemoProject({ projectDir, force: true });
|
||||
if (preservedConfig !== null) {
|
||||
await writeFile(result.configPath, preservedConfig, 'utf-8');
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
async function readExistingConfig(configPath: string): Promise<string | null> {
|
||||
try {
|
||||
return await readFile(configPath, 'utf-8');
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function demoConfig(databasePath: string): string {
|
||||
return [
|
||||
'project: klo-demo-orbit',
|
||||
'connections:',
|
||||
` ${DEMO_CONNECTION_ID}:`,
|
||||
' driver: sqlite',
|
||||
` path: ${JSON.stringify(databasePath)}`,
|
||||
' readonly: true',
|
||||
'storage:',
|
||||
' state: sqlite',
|
||||
' search: sqlite-fts5',
|
||||
' git:',
|
||||
' auto_commit: true',
|
||||
' author: klo <klo@example.com>',
|
||||
'llm:',
|
||||
' provider:',
|
||||
' backend: anthropic',
|
||||
' anthropic:',
|
||||
' api_key: env:ANTHROPIC_API_KEY',
|
||||
' models:',
|
||||
' default: claude-sonnet-4-6',
|
||||
'ingest:',
|
||||
' adapters:',
|
||||
` - ${DEMO_ADAPTER}`,
|
||||
' embeddings:',
|
||||
' backend: none',
|
||||
' dimensions: 8',
|
||||
' workUnits:',
|
||||
' stepBudget: 40',
|
||||
' maxConcurrency: 1',
|
||||
' failureMode: continue',
|
||||
'',
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
async function copyPackagedReplay(projectDir: string): Promise<string> {
|
||||
const replayDir = join(projectDir, 'replays');
|
||||
await mkdir(replayDir, { recursive: true });
|
||||
const replayPath = join(replayDir, DEMO_REPLAY_FILE);
|
||||
await copyFile(join(assetDir(), DEMO_REPLAY_FILE), replayPath);
|
||||
return replayPath;
|
||||
}
|
||||
|
||||
async function assertPackagedBaseAssetsPresent(): Promise<void> {
|
||||
const missing: string[] = [];
|
||||
for (const relativePath of REQUIRED_PACKAGED_BASE_ASSET_PATHS) {
|
||||
if (!(await exists(join(assetDir(), relativePath)))) {
|
||||
missing.push(relativePath);
|
||||
}
|
||||
}
|
||||
if (missing.length > 0) {
|
||||
throw new Error(`Packaged demo assets are incomplete: missing ${missing.join(', ')}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function assertPackagedSeededAssetsPresent(): Promise<void> {
|
||||
const missing: string[] = [];
|
||||
for (const relativePath of REQUIRED_SEEDED_ASSET_PATHS) {
|
||||
if (!(await exists(join(assetDir(), relativePath)))) {
|
||||
missing.push(relativePath);
|
||||
}
|
||||
}
|
||||
if (missing.length > 0) {
|
||||
throw new Error(`Packaged seeded demo assets are incomplete: missing ${missing.join(', ')}`);
|
||||
}
|
||||
}
|
||||
|
||||
export async function ensureDemoProject(options: EnsureDemoProjectOptions): Promise<DemoProjectResult> {
|
||||
const projectDir = resolve(options.projectDir);
|
||||
const configPath = join(projectDir, 'klo.yaml');
|
||||
if (!options.force && (await exists(configPath))) {
|
||||
throw new Error(`Demo project already exists at ${projectDir}; pass --force to recreate it`);
|
||||
}
|
||||
|
||||
await assertPackagedBaseAssetsPresent();
|
||||
|
||||
if (options.force) {
|
||||
await rm(projectDir, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
await mkdir(projectDir, { recursive: true });
|
||||
for (const relativeDir of ['reports', 'semantic-layer', 'knowledge', 'replays', 'raw-sources', 'links']) {
|
||||
await mkdir(join(projectDir, relativeDir), { recursive: true });
|
||||
}
|
||||
|
||||
const databasePath = join(projectDir, 'demo.db');
|
||||
await copyFile(join(assetDir(), 'demo.db'), databasePath);
|
||||
await writeFile(join(projectDir, 'state.sqlite'), '', { flag: 'a' });
|
||||
await copyFile(join(assetDir(), 'manifest.json'), join(projectDir, 'manifest.json'));
|
||||
const replayPath = await copyPackagedReplay(projectDir);
|
||||
await writeFile(configPath, demoConfig(databasePath), 'utf-8');
|
||||
|
||||
return { projectDir, configPath, databasePath, replayPath };
|
||||
}
|
||||
|
||||
async function copyDirIfExists(src: string, dest: string): Promise<void> {
|
||||
if (await exists(src)) {
|
||||
await cp(src, dest, { recursive: true });
|
||||
}
|
||||
}
|
||||
|
||||
async function copySeededAssetDirectories(projectDir: string): Promise<void> {
|
||||
const src = assetDir();
|
||||
const dest = resolve(projectDir);
|
||||
|
||||
await Promise.all([
|
||||
copyDirIfExists(join(src, 'semantic-layer'), join(dest, 'semantic-layer')),
|
||||
copyDirIfExists(join(src, 'knowledge'), join(dest, 'knowledge')),
|
||||
copyDirIfExists(join(src, 'raw-sources'), join(dest, 'raw-sources')),
|
||||
copyDirIfExists(join(src, 'links'), join(dest, 'links')),
|
||||
copyDirIfExists(join(src, 'reports'), join(dest, 'reports')),
|
||||
]);
|
||||
}
|
||||
|
||||
export async function ensureSeededDemoProject(options: EnsureDemoProjectOptions): Promise<DemoProjectResult> {
|
||||
await assertPackagedSeededAssetsPresent();
|
||||
const projectDir = resolve(options.projectDir);
|
||||
const result = await ensureDemoProject(options).catch((error) => {
|
||||
if (!options.force && error instanceof Error && error.message.includes('Demo project already exists')) {
|
||||
return {
|
||||
projectDir,
|
||||
configPath: join(projectDir, 'klo.yaml'),
|
||||
databasePath: join(projectDir, 'demo.db'),
|
||||
replayPath: join(projectDir, 'replays', DEMO_REPLAY_FILE),
|
||||
};
|
||||
}
|
||||
throw error;
|
||||
});
|
||||
|
||||
await copySeededAssetDirectories(result.projectDir);
|
||||
return result;
|
||||
}
|
||||
|
||||
export async function loadPackagedDemoReplay(): Promise<MemoryFlowReplayInput> {
|
||||
const replay = await loadDemoReplayFile(join(assetDir(), DEMO_REPLAY_FILE));
|
||||
return {
|
||||
...replay,
|
||||
metadata: {
|
||||
schemaVersion: 1,
|
||||
mode: replay.metadata?.mode ?? 'seeded',
|
||||
origin: 'packaged',
|
||||
timing: replay.metadata?.timing ?? 'prebuilt',
|
||||
capturedAt: replay.metadata?.capturedAt ?? null,
|
||||
sourceReportId: replay.metadata?.sourceReportId ?? 'demo-seeded-report',
|
||||
sourceReportPath: replay.metadata?.sourceReportPath ?? `reports/seeded-demo-report.json`,
|
||||
fallbackReason: null,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export async function loadProjectDemoReplay(projectDir: string): Promise<MemoryFlowReplayInput> {
|
||||
const latest = await loadLatestDemoReplay(projectDir);
|
||||
if (latest) {
|
||||
return latest;
|
||||
}
|
||||
|
||||
const replayPath = join(resolve(projectDir), 'replays', DEMO_REPLAY_FILE);
|
||||
if (!(await exists(replayPath))) {
|
||||
await mkdir(dirname(replayPath), { recursive: true });
|
||||
await copyPackagedReplay(resolve(projectDir));
|
||||
}
|
||||
return loadPackagedDemoReplay();
|
||||
}
|
||||
201
packages/cli/src/demo-full.test.ts
Normal file
201
packages/cli/src/demo-full.test.ts
Normal file
|
|
@ -0,0 +1,201 @@
|
|||
import { mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import type { IngestReportSnapshot, LocalIngestResult, RunLocalIngestOptions } from '@klo/context/ingest';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { DEMO_ADAPTER, DEMO_CONNECTION_ID, DEMO_FULL_JOB_ID, ensureDemoProject } from './demo-assets.js';
|
||||
import {
|
||||
assertFullDemoCredentials,
|
||||
buildFullDemoReplay,
|
||||
formatFullDemoSummary,
|
||||
fullDemoCredentialStatus,
|
||||
runDemoFull,
|
||||
} from './demo-full.js';
|
||||
|
||||
function fakeFullReport(): IngestReportSnapshot {
|
||||
return {
|
||||
id: 'report-full',
|
||||
runId: 'run-full',
|
||||
jobId: DEMO_FULL_JOB_ID,
|
||||
connectionId: DEMO_CONNECTION_ID,
|
||||
sourceKey: DEMO_ADAPTER,
|
||||
createdAt: '2026-05-01T00:00:00.000Z',
|
||||
body: {
|
||||
syncId: 'sync-full',
|
||||
diffSummary: { added: 7, modified: 0, deleted: 0, unchanged: 0 },
|
||||
commitSha: null,
|
||||
workUnits: [
|
||||
{
|
||||
unitKey: 'accounts',
|
||||
rawFiles: ['accounts.schema.json'],
|
||||
status: 'success',
|
||||
actions: [
|
||||
{ target: 'wiki', type: 'created', key: 'knowledge/accounts.md', detail: 'account lifecycle context' },
|
||||
{ target: 'sl', type: 'created', key: 'orbit_demo.accounts', detail: 'accounts semantic source' },
|
||||
],
|
||||
touchedSlSources: [{ connectionId: 'orbit_demo', sourceName: 'orbit_demo.accounts' }],
|
||||
},
|
||||
],
|
||||
failedWorkUnits: [],
|
||||
reconciliationSkipped: false,
|
||||
conflictsResolved: [],
|
||||
evictionsApplied: [],
|
||||
unmappedFallbacks: [],
|
||||
evictionInputs: [],
|
||||
unresolvedCards: [],
|
||||
supersededBy: null,
|
||||
overrideOf: null,
|
||||
provenanceRows: [
|
||||
{
|
||||
rawPath: 'accounts.schema.json',
|
||||
artifactKind: 'wiki',
|
||||
artifactKey: 'knowledge/accounts.md',
|
||||
actionType: 'wiki_written',
|
||||
},
|
||||
{
|
||||
rawPath: 'accounts.schema.json',
|
||||
artifactKind: 'sl',
|
||||
artifactKey: 'orbit_demo.accounts',
|
||||
actionType: 'source_created',
|
||||
},
|
||||
],
|
||||
toolTranscripts: [],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe('full demo helpers', () => {
|
||||
let tempDir: string;
|
||||
let projectDir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'klo-demo-full-'));
|
||||
projectDir = join(tempDir, 'demo');
|
||||
await ensureDemoProject({ projectDir, force: false });
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(tempDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('fails full mode with exact Anthropic env guidance when the key is missing', async () => {
|
||||
const project = await import('@klo/context/project').then((mod) => mod.loadKloProject({ projectDir }));
|
||||
|
||||
expect(() => assertFullDemoCredentials(project, {})).toThrow(
|
||||
'klo setup demo --mode full needs ANTHROPIC_API_KEY. Export ANTHROPIC_API_KEY and rerun `klo setup demo --mode full --no-input`, or run `klo setup demo --mode seeded --no-input` without credentials.',
|
||||
);
|
||||
});
|
||||
|
||||
it('respects an existing gateway provider project for full mode', async () => {
|
||||
await writeFile(
|
||||
join(projectDir, 'klo.yaml'),
|
||||
[
|
||||
'project: klo-demo-orbit',
|
||||
'connections:',
|
||||
' orbit_demo:',
|
||||
' driver: sqlite',
|
||||
` path: ${JSON.stringify(join(projectDir, 'demo.db'))}`,
|
||||
'llm:',
|
||||
' provider:',
|
||||
' backend: gateway',
|
||||
' models:',
|
||||
' default: anthropic/claude-sonnet-4-6',
|
||||
'ingest:',
|
||||
' adapters:',
|
||||
' - live-database',
|
||||
' embeddings:',
|
||||
' backend: none',
|
||||
'',
|
||||
].join('\n'),
|
||||
'utf-8',
|
||||
);
|
||||
const project = await import('@klo/context/project').then((mod) => mod.loadKloProject({ projectDir }));
|
||||
|
||||
expect(() => assertFullDemoCredentials(project, {})).not.toThrow();
|
||||
expect(fullDemoCredentialStatus(project, {})).toEqual({ status: 'ready' });
|
||||
});
|
||||
|
||||
it('reports full-demo credential status without throwing', async () => {
|
||||
const project = await import('@klo/context/project').then((mod) => mod.loadKloProject({ projectDir }));
|
||||
|
||||
expect(fullDemoCredentialStatus(project, {})).toEqual({ status: 'missing-anthropic-key' });
|
||||
expect(fullDemoCredentialStatus(project, { ANTHROPIC_API_KEY: 'sk-ant-test' })).toEqual({ status: 'ready' }); // pragma: allowlist secret
|
||||
|
||||
await writeFile(
|
||||
join(projectDir, 'klo.yaml'),
|
||||
[
|
||||
'project: klo-demo-orbit',
|
||||
'connections:',
|
||||
' orbit_demo:',
|
||||
' driver: sqlite',
|
||||
` path: ${JSON.stringify(join(projectDir, 'demo.db'))}`,
|
||||
'ingest:',
|
||||
' adapters:',
|
||||
' - live-database',
|
||||
'',
|
||||
].join('\n'),
|
||||
'utf-8',
|
||||
);
|
||||
const disabledProject = await import('@klo/context/project').then((mod) => mod.loadKloProject({ projectDir }));
|
||||
expect(fullDemoCredentialStatus(disabledProject, {})).toEqual({ status: 'unsupported-provider', provider: 'none' });
|
||||
});
|
||||
|
||||
it('runs scan first and then full ingest with the canonical demo connection', async () => {
|
||||
const report = fakeFullReport();
|
||||
const runLocalScan = vi.fn().mockResolvedValue({
|
||||
report: {
|
||||
runId: 'scan-run',
|
||||
connectionId: DEMO_CONNECTION_ID,
|
||||
driver: 'sqlite',
|
||||
mode: 'structural',
|
||||
syncId: 'sync-scan',
|
||||
diffSummary: { tablesAdded: 7, tablesModified: 0, tablesDeleted: 0, tablesUnchanged: 0 },
|
||||
artifactPaths: { rawSourcesDir: 'raw-sources/orbit_demo/live-database/sync-scan', manifestShards: [], reportPath: 'scan-report.json' },
|
||||
},
|
||||
});
|
||||
const runLocalIngest = vi.fn(async (options: RunLocalIngestOptions): Promise<LocalIngestResult> => {
|
||||
expect(options.adapter).toBe(DEMO_ADAPTER);
|
||||
expect(options.connectionId).toBe(DEMO_CONNECTION_ID);
|
||||
expect(options.jobId).toBe(DEMO_FULL_JOB_ID);
|
||||
expect(options.memoryFlow?.snapshot()).toMatchObject({ runId: DEMO_FULL_JOB_ID, status: 'running' });
|
||||
options.memoryFlow?.emit({ type: 'source_acquired', adapter: DEMO_ADAPTER, trigger: 'demo_full', fileCount: 7 });
|
||||
return { result: { ok: true } as never, report };
|
||||
});
|
||||
const snapshots: unknown[] = [];
|
||||
|
||||
const result = await runDemoFull({
|
||||
projectDir,
|
||||
env: { ANTHROPIC_API_KEY: 'sk-ant-test' }, // pragma: allowlist secret
|
||||
runLocalScan,
|
||||
runLocalIngest,
|
||||
onMemoryFlowChange: (snapshot) => snapshots.push(snapshot),
|
||||
});
|
||||
|
||||
expect(runLocalScan).toHaveBeenCalledTimes(1);
|
||||
expect(runLocalIngest).toHaveBeenCalledTimes(1);
|
||||
expect(result.report).toBe(report);
|
||||
expect(result.replay.runId).toBe('run-full');
|
||||
expect(snapshots).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('builds replay and plain summary from the full report', () => {
|
||||
const report = fakeFullReport();
|
||||
const replay = buildFullDemoReplay(report);
|
||||
const summary = formatFullDemoSummary(report);
|
||||
|
||||
expect(replay).toMatchObject({
|
||||
runId: 'run-full',
|
||||
connectionId: DEMO_CONNECTION_ID,
|
||||
adapter: DEMO_ADAPTER,
|
||||
status: 'done',
|
||||
});
|
||||
expect(summary).toContain('Full demo ingest: done');
|
||||
expect(summary).toContain('Saved memory: 1 wiki, 1 semantic layer');
|
||||
expect(summary).toContain('Provenance rows: 2');
|
||||
expect(summary).toContain('Next: klo setup demo inspect');
|
||||
expect(summary).toContain('Shows the files, semantic-layer sources, and memory KLO just produced.');
|
||||
expect(summary).toContain('Next: klo setup demo replay');
|
||||
expect(summary).toContain('Replays the same visual story without calling the LLM again.');
|
||||
expect(summary).not.toContain('--viz');
|
||||
});
|
||||
});
|
||||
213
packages/cli/src/demo-full.ts
Normal file
213
packages/cli/src/demo-full.ts
Normal file
|
|
@ -0,0 +1,213 @@
|
|||
import { resolveKloConfigReference } from '@klo/context/core';
|
||||
import {
|
||||
createMemoryFlowLiveBuffer,
|
||||
ingestReportToMemoryFlowReplay,
|
||||
runLocalIngest,
|
||||
type IngestReportSnapshot,
|
||||
type LocalIngestResult,
|
||||
type MemoryFlowReplayInput,
|
||||
type RunLocalIngestOptions,
|
||||
} from '@klo/context/ingest';
|
||||
import { loadKloProject, type KloLocalProject } from '@klo/context/project';
|
||||
import { runLocalScan, type LocalScanRunResult } from '@klo/context/scan';
|
||||
import { DEMO_ADAPTER, DEMO_CONNECTION_ID, DEMO_FULL_JOB_ID, ensureDemoProject } from './demo-assets.js';
|
||||
import { runDemoScan } from './demo-scan.js';
|
||||
import { createKloCliLocalIngestAdapters } from './local-adapters.js';
|
||||
import { formatNextStepLines } from './next-steps.js';
|
||||
|
||||
interface DemoFullOptions {
|
||||
projectDir: string;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
runLocalScan?: typeof runLocalScan;
|
||||
runLocalIngest?: typeof runLocalIngest;
|
||||
onMemoryFlowChange?: (snapshot: MemoryFlowReplayInput) => void;
|
||||
}
|
||||
|
||||
export interface DemoFullResult {
|
||||
project: KloLocalProject;
|
||||
scan: LocalScanRunResult;
|
||||
ingest: LocalIngestResult;
|
||||
report: IngestReportSnapshot;
|
||||
replay: MemoryFlowReplayInput;
|
||||
}
|
||||
|
||||
type FullDemoCredentialStatus =
|
||||
| { status: 'ready' }
|
||||
| { status: 'missing-anthropic-key' }
|
||||
| { status: 'unsupported-provider'; provider: string };
|
||||
|
||||
async function ensureDemoProjectForReuse(projectDir: string): Promise<void> {
|
||||
await ensureDemoProject({ projectDir, force: false }).catch((error) => {
|
||||
if (error instanceof Error && error.message.includes('Demo project already exists')) {
|
||||
return;
|
||||
}
|
||||
throw error;
|
||||
});
|
||||
}
|
||||
|
||||
function savedCounts(report: IngestReportSnapshot): { wikiCount: number; slCount: number } {
|
||||
const actions = report.body.workUnits.flatMap((workUnit) => workUnit.actions);
|
||||
return {
|
||||
wikiCount: actions.filter((action) => action.target === 'wiki').length,
|
||||
slCount: actions.filter((action) => action.target === 'sl').length,
|
||||
};
|
||||
}
|
||||
|
||||
export function fullDemoCredentialStatus(
|
||||
project: KloLocalProject,
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
): FullDemoCredentialStatus {
|
||||
const llm = project.config.llm;
|
||||
if (llm.provider.backend === 'none') {
|
||||
return { status: 'unsupported-provider', provider: llm.provider.backend };
|
||||
}
|
||||
|
||||
if (llm.provider.backend === 'anthropic' && !resolveKloConfigReference(llm.provider.anthropic?.api_key, env)) {
|
||||
return { status: 'missing-anthropic-key' };
|
||||
}
|
||||
|
||||
return { status: 'ready' };
|
||||
}
|
||||
|
||||
export function assertFullDemoCredentials(project: KloLocalProject, env: NodeJS.ProcessEnv = process.env): void {
|
||||
const llm = project.config.llm;
|
||||
const status = fullDemoCredentialStatus(project, env);
|
||||
if (status.status === 'ready') {
|
||||
return;
|
||||
}
|
||||
|
||||
if (status.status === 'unsupported-provider') {
|
||||
throw new Error(
|
||||
'klo setup demo --mode full requires llm.provider.backend: anthropic, vertex, or gateway. Run `klo setup demo init --force --no-input` to recreate the demo config, or run `klo setup demo --mode seeded --no-input` without credentials.',
|
||||
);
|
||||
}
|
||||
|
||||
if (llm.provider.backend === 'anthropic') {
|
||||
throw new Error(
|
||||
'klo setup demo --mode full needs ANTHROPIC_API_KEY. Export ANTHROPIC_API_KEY and rerun `klo setup demo --mode full --no-input`, or run `klo setup demo --mode seeded --no-input` without credentials.',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export function buildFullDemoReplay(report: IngestReportSnapshot): MemoryFlowReplayInput {
|
||||
return ingestReportToMemoryFlowReplay(report, { provenanceRowCount: report.body.provenanceRows.length });
|
||||
}
|
||||
|
||||
function initialFullReplay(projectDir: string): MemoryFlowReplayInput {
|
||||
return {
|
||||
runId: DEMO_FULL_JOB_ID,
|
||||
connectionId: DEMO_CONNECTION_ID,
|
||||
adapter: DEMO_ADAPTER,
|
||||
status: 'running',
|
||||
sourceDir: `${projectDir}/raw-sources/${DEMO_CONNECTION_ID}/${DEMO_ADAPTER}`,
|
||||
syncId: 'pending',
|
||||
errors: [],
|
||||
events: [],
|
||||
plannedWorkUnits: [],
|
||||
details: { actions: [], provenance: [], transcripts: [] },
|
||||
};
|
||||
}
|
||||
|
||||
export async function runDemoFull(options: DemoFullOptions): Promise<DemoFullResult> {
|
||||
await ensureDemoProjectForReuse(options.projectDir);
|
||||
const project = await loadKloProject({ projectDir: options.projectDir });
|
||||
assertFullDemoCredentials(project, options.env);
|
||||
|
||||
const { result: scan } = await runDemoScan({
|
||||
projectDir: project.projectDir,
|
||||
jobId: 'demo-full-scan',
|
||||
...(options.runLocalScan ? { runLocalScan: options.runLocalScan } : {}),
|
||||
});
|
||||
|
||||
const memoryFlow = options.onMemoryFlowChange
|
||||
? createMemoryFlowLiveBuffer(initialFullReplay(project.projectDir), { onChange: options.onMemoryFlowChange })
|
||||
: undefined;
|
||||
const executeLocalIngest = options.runLocalIngest ?? runLocalIngest;
|
||||
const ingest = await executeLocalIngest({
|
||||
project,
|
||||
adapters: createKloCliLocalIngestAdapters(project),
|
||||
adapter: DEMO_ADAPTER,
|
||||
connectionId: DEMO_CONNECTION_ID,
|
||||
trigger: 'manual_resync',
|
||||
jobId: DEMO_FULL_JOB_ID,
|
||||
...(memoryFlow ? { memoryFlow } : {}),
|
||||
} satisfies RunLocalIngestOptions);
|
||||
|
||||
return {
|
||||
project,
|
||||
scan,
|
||||
ingest,
|
||||
report: ingest.report,
|
||||
replay: buildFullDemoReplay(ingest.report),
|
||||
};
|
||||
}
|
||||
|
||||
export function formatFullDemoSummary(report: IngestReportSnapshot): string {
|
||||
const counts = savedCounts(report);
|
||||
return [
|
||||
'Full demo ingest: done',
|
||||
`Report: ${report.id}`,
|
||||
`Run: ${report.runId}`,
|
||||
`Job: ${report.jobId}`,
|
||||
`Sync: ${report.body.syncId}`,
|
||||
`Saved memory: ${counts.wikiCount} wiki, ${counts.slCount} semantic layer`,
|
||||
`Provenance rows: ${report.body.provenanceRows.length}`,
|
||||
'Next: klo setup demo inspect',
|
||||
' Shows the files, semantic-layer sources, and memory KLO just produced.',
|
||||
'Next: klo setup demo replay',
|
||||
' Replays the same visual story without calling the LLM again.',
|
||||
'',
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
const ADAPTER_PREFIXES = ['live_database_', 'metabase_', 'looker_', 'lookml_', 'metricflow_', 'notion_', 'historic_sql_', 'dbt_descriptions_'];
|
||||
|
||||
function humanizeUnitKeyForReport(unitKey: string): string {
|
||||
let key = unitKey.replace(/-/g, '_');
|
||||
for (const prefix of ADAPTER_PREFIXES) {
|
||||
if (key.startsWith(prefix)) { key = key.slice(prefix.length); break; }
|
||||
}
|
||||
return key.replace(/_/g, ' ');
|
||||
}
|
||||
|
||||
export function formatCleanDemoSummary(report: IngestReportSnapshot, projectDir: string): string {
|
||||
const counts = savedCounts(report);
|
||||
const workUnits = report.body.workUnits;
|
||||
const conflictCount = report.body.conflictsResolved.length;
|
||||
const areasAnalyzed = workUnits.filter((wu) => wu.actions.length > 0).length;
|
||||
|
||||
const lines: string[] = ['', '★ KLO finished ingesting your data', ''];
|
||||
|
||||
if (areasAnalyzed > 0) {
|
||||
lines.push(` ✓ Analyzed ${areasAnalyzed} business area${areasAnalyzed === 1 ? '' : 's'}`);
|
||||
}
|
||||
if (!report.body.reconciliationSkipped) {
|
||||
lines.push(` ✓ Reconciled — ${conflictCount > 0 ? `${conflictCount} conflict${conflictCount === 1 ? '' : 's'} resolved` : 'no conflicts'}`);
|
||||
}
|
||||
lines.push('');
|
||||
|
||||
if (counts.slCount > 0 || counts.wikiCount > 0) {
|
||||
lines.push(' KLO created:');
|
||||
if (counts.slCount > 0) lines.push(` 📊 ${counts.slCount} query definition${counts.slCount === 1 ? '' : 's'} — so agents can write accurate SQL for your data`);
|
||||
if (counts.wikiCount > 0) lines.push(` 📝 ${counts.wikiCount} knowledge page${counts.wikiCount === 1 ? '' : 's'} — so agents understand your business context`);
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
const memoryFlow = report.body.memoryFlow;
|
||||
if (memoryFlow) {
|
||||
for (const detail of memoryFlow.details.actions) {
|
||||
if (!detail.summary) continue;
|
||||
const icon = detail.target === 'sl' ? '📊' : '📝';
|
||||
lines.push(` ${icon} ${detail.summary}`);
|
||||
}
|
||||
}
|
||||
|
||||
lines.push('');
|
||||
lines.push(' What to do next:');
|
||||
lines.push(...formatNextStepLines());
|
||||
lines.push('');
|
||||
lines.push(` Your KLO project files are at: ${projectDir}`);
|
||||
lines.push('');
|
||||
|
||||
return lines.join('\n');
|
||||
}
|
||||
127
packages/cli/src/demo-interaction.test.ts
Normal file
127
packages/cli/src/demo-interaction.test.ts
Normal file
|
|
@ -0,0 +1,127 @@
|
|||
import { mkdtemp, rm } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { ensureDemoProject } from './demo-assets.js';
|
||||
import {
|
||||
chooseDemoProjectForInteractiveRun,
|
||||
createTestDemoPromptAdapter,
|
||||
resolveFullCredentialDecision,
|
||||
} from './demo-interaction.js';
|
||||
|
||||
function io(isTTY: boolean) {
|
||||
return {
|
||||
stdin: { isTTY },
|
||||
stdout: { isTTY, write: vi.fn() },
|
||||
stderr: { write: vi.fn() },
|
||||
};
|
||||
}
|
||||
|
||||
describe('demo interaction decisions', () => {
|
||||
let tempDir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'klo-demo-interaction-'));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(tempDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('reuses a valid project without prompting in no-input mode', async () => {
|
||||
await ensureDemoProject({ projectDir: tempDir, force: false });
|
||||
|
||||
await expect(
|
||||
chooseDemoProjectForInteractiveRun({
|
||||
projectDir: tempDir,
|
||||
inputMode: 'disabled',
|
||||
io: io(false),
|
||||
prompts: createTestDemoPromptAdapter({ choices: [] }),
|
||||
}),
|
||||
).resolves.toEqual({ action: 'use', projectDir: tempDir, reset: false });
|
||||
});
|
||||
|
||||
it('fails corrupted projects in no-input mode with reset guidance', async () => {
|
||||
await ensureDemoProject({ projectDir: tempDir, force: false });
|
||||
await rm(join(tempDir, 'demo.db'), { force: true });
|
||||
|
||||
await expect(
|
||||
chooseDemoProjectForInteractiveRun({
|
||||
projectDir: tempDir,
|
||||
inputMode: 'disabled',
|
||||
io: io(false),
|
||||
prompts: createTestDemoPromptAdapter({ choices: [] }),
|
||||
}),
|
||||
).rejects.toThrow(
|
||||
`Demo project is not ready at ${tempDir}: missing demo.db. Run klo setup demo reset --project-dir ${tempDir} --force --no-input`,
|
||||
);
|
||||
});
|
||||
|
||||
it('lets interactive users reset a corrupted project', async () => {
|
||||
await ensureDemoProject({ projectDir: tempDir, force: false });
|
||||
await rm(join(tempDir, 'demo.db'), { force: true });
|
||||
|
||||
await expect(
|
||||
chooseDemoProjectForInteractiveRun({
|
||||
projectDir: tempDir,
|
||||
io: io(true),
|
||||
prompts: createTestDemoPromptAdapter({ choices: ['reset'], confirms: [true] }),
|
||||
}),
|
||||
).resolves.toEqual({ action: 'use', projectDir: tempDir, reset: true });
|
||||
});
|
||||
|
||||
it('lets interactive users choose another project directory', async () => {
|
||||
await ensureDemoProject({ projectDir: tempDir, force: false });
|
||||
const otherDir = join(tempDir, 'other-demo');
|
||||
|
||||
await expect(
|
||||
chooseDemoProjectForInteractiveRun({
|
||||
projectDir: tempDir,
|
||||
io: io(true),
|
||||
prompts: createTestDemoPromptAdapter({ choices: ['other'], texts: [otherDir] }),
|
||||
}),
|
||||
).resolves.toEqual({ action: 'use', projectDir: otherDir, reset: false });
|
||||
});
|
||||
|
||||
it('uses a pasted Anthropic key only for the returned process env', async () => {
|
||||
// pragma: allowlist secret
|
||||
const prompts = createTestDemoPromptAdapter({ choices: ['process_key'], passwords: ['sk-ant-process'] });
|
||||
|
||||
await expect(
|
||||
resolveFullCredentialDecision({
|
||||
needsAnthropicKey: true,
|
||||
inputMode: 'auto',
|
||||
io: io(true),
|
||||
env: {},
|
||||
prompts,
|
||||
}),
|
||||
).resolves.toEqual({
|
||||
action: 'full',
|
||||
env: { ANTHROPIC_API_KEY: 'sk-ant-process' }, // pragma: allowlist secret
|
||||
});
|
||||
});
|
||||
|
||||
it('lets interactive users explicitly choose seeded mode when the key is missing', async () => {
|
||||
await expect(
|
||||
resolveFullCredentialDecision({
|
||||
needsAnthropicKey: true,
|
||||
inputMode: 'auto',
|
||||
io: io(true),
|
||||
env: {},
|
||||
prompts: createTestDemoPromptAdapter({ choices: ['seeded'] }),
|
||||
}),
|
||||
).resolves.toEqual({ action: 'run-mode', mode: 'seeded' });
|
||||
});
|
||||
|
||||
it('does not prompt when input is disabled', async () => {
|
||||
await expect(
|
||||
resolveFullCredentialDecision({
|
||||
needsAnthropicKey: true,
|
||||
inputMode: 'disabled',
|
||||
io: io(false),
|
||||
env: {},
|
||||
prompts: createTestDemoPromptAdapter({ choices: ['seeded'] }),
|
||||
}),
|
||||
).resolves.toEqual({ action: 'full', env: {} });
|
||||
});
|
||||
});
|
||||
202
packages/cli/src/demo-interaction.ts
Normal file
202
packages/cli/src/demo-interaction.ts
Normal file
|
|
@ -0,0 +1,202 @@
|
|||
import { cancel, confirm, isCancel, password, select, text } from '@clack/prompts';
|
||||
import type { Option as ClackOption } from '@clack/prompts';
|
||||
import { resolve } from 'node:path';
|
||||
import { inspectDemoProjectState } from './demo-assets.js';
|
||||
import type { KloDemoInputMode } from './demo.js';
|
||||
import { withMenuOptionsSpacing } from './prompt-navigation.js';
|
||||
|
||||
type DemoPromptOption<T extends string> = ClackOption<T>;
|
||||
|
||||
export interface DemoPromptAdapter {
|
||||
select<T extends string>(options: { message: string; options: Array<DemoPromptOption<T>> }): Promise<T>;
|
||||
confirm(options: { message: string; initialValue?: boolean }): Promise<boolean>;
|
||||
password(options: { message: string }): Promise<string>;
|
||||
text(options: { message: string; placeholder?: string }): Promise<string>;
|
||||
cancel(message: string): void;
|
||||
}
|
||||
|
||||
interface DemoInteractiveIo {
|
||||
stdin?: { isTTY?: boolean };
|
||||
stdout: { isTTY?: boolean };
|
||||
}
|
||||
|
||||
type DemoProjectDecision =
|
||||
| { action: 'use'; projectDir: string; reset: boolean }
|
||||
| { action: 'cancel' };
|
||||
|
||||
type FullCredentialDecision =
|
||||
| { action: 'full'; env: NodeJS.ProcessEnv }
|
||||
| { action: 'run-mode'; mode: 'seeded' | 'replay' }
|
||||
| { action: 'cancel' };
|
||||
|
||||
function isInteractive(inputMode: KloDemoInputMode | undefined, io: DemoInteractiveIo): boolean {
|
||||
return inputMode !== 'disabled' && io.stdin?.isTTY === true && io.stdout.isTTY === true;
|
||||
}
|
||||
|
||||
function cloneEnv(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv {
|
||||
return { ...env };
|
||||
}
|
||||
|
||||
function ensureNotCancelled<T>(value: T | symbol, prompts: Pick<DemoPromptAdapter, 'cancel'>): T {
|
||||
if (isCancel(value)) {
|
||||
prompts.cancel('Demo cancelled.');
|
||||
throw new Error('Demo cancelled.');
|
||||
}
|
||||
return value as T;
|
||||
}
|
||||
|
||||
export function createClackDemoPromptAdapter(): DemoPromptAdapter {
|
||||
return {
|
||||
async select<T extends string>(options: { message: string; options: Array<DemoPromptOption<T>> }): Promise<T> {
|
||||
return ensureNotCancelled(await select(withMenuOptionsSpacing(options)), this);
|
||||
},
|
||||
async confirm(options: { message: string; initialValue?: boolean }): Promise<boolean> {
|
||||
return ensureNotCancelled(await confirm(options), this);
|
||||
},
|
||||
async password(options: { message: string }): Promise<string> {
|
||||
return ensureNotCancelled(await password(options), this);
|
||||
},
|
||||
async text(options: { message: string; placeholder?: string }): Promise<string> {
|
||||
return ensureNotCancelled(await text(options), this);
|
||||
},
|
||||
cancel(message: string): void {
|
||||
cancel(message);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function createTestDemoPromptAdapter(options: {
|
||||
choices?: string[];
|
||||
confirms?: boolean[];
|
||||
passwords?: string[];
|
||||
texts?: string[];
|
||||
}): DemoPromptAdapter {
|
||||
const choices = [...(options.choices ?? [])];
|
||||
const confirms = [...(options.confirms ?? [])];
|
||||
const passwords = [...(options.passwords ?? [])];
|
||||
const texts = [...(options.texts ?? [])];
|
||||
|
||||
return {
|
||||
async select<T extends string>(): Promise<T> {
|
||||
return choices.shift() as T;
|
||||
},
|
||||
async confirm(): Promise<boolean> {
|
||||
return confirms.shift() ?? false;
|
||||
},
|
||||
async password(): Promise<string> {
|
||||
return passwords.shift() ?? '';
|
||||
},
|
||||
async text(): Promise<string> {
|
||||
return texts.shift() ?? '';
|
||||
},
|
||||
cancel(): void {
|
||||
return;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export async function chooseDemoProjectForInteractiveRun(options: {
|
||||
projectDir: string;
|
||||
inputMode?: KloDemoInputMode;
|
||||
io: DemoInteractiveIo;
|
||||
prompts?: DemoPromptAdapter;
|
||||
}): Promise<DemoProjectDecision> {
|
||||
const prompts = options.prompts ?? createClackDemoPromptAdapter();
|
||||
const projectDir = resolve(options.projectDir);
|
||||
const state = await inspectDemoProjectState(projectDir);
|
||||
|
||||
if (!isInteractive(options.inputMode, options.io)) {
|
||||
if (state.status === 'corrupt') {
|
||||
throw new Error(
|
||||
`Demo project is not ready at ${projectDir}: missing ${state.missing.join(', ')}. Run klo setup demo reset --project-dir ${projectDir} --force --no-input`,
|
||||
);
|
||||
}
|
||||
return { action: 'use', projectDir, reset: false };
|
||||
}
|
||||
|
||||
if (state.status === 'missing') {
|
||||
return { action: 'use', projectDir, reset: false };
|
||||
}
|
||||
|
||||
const choices =
|
||||
state.status === 'ready'
|
||||
? [
|
||||
{ value: 'reuse', label: 'Reuse existing demo project' },
|
||||
{ value: 'reset', label: 'Reset demo project' },
|
||||
{ value: 'other', label: 'Choose another directory' },
|
||||
{ value: 'cancel', label: 'Cancel' },
|
||||
]
|
||||
: [
|
||||
{ value: 'reset', label: 'Reset corrupted demo project', hint: `Missing ${state.missing.join(', ')}` },
|
||||
{ value: 'other', label: 'Choose another directory' },
|
||||
{ value: 'cancel', label: 'Cancel' },
|
||||
];
|
||||
|
||||
const choice = await prompts.select({
|
||||
message: state.status === 'ready' ? `Demo project exists at ${projectDir}` : `Demo project is not ready at ${projectDir}`,
|
||||
options: choices,
|
||||
});
|
||||
|
||||
if (choice === 'cancel') {
|
||||
prompts.cancel('Demo cancelled.');
|
||||
return { action: 'cancel' };
|
||||
}
|
||||
|
||||
if (choice === 'other') {
|
||||
const nextProjectDir = await prompts.text({
|
||||
message: 'Demo project directory',
|
||||
placeholder: projectDir,
|
||||
});
|
||||
return { action: 'use', projectDir: resolve(nextProjectDir), reset: false };
|
||||
}
|
||||
|
||||
if (choice === 'reset') {
|
||||
const confirmed = await prompts.confirm({
|
||||
message: `Recreate ${projectDir}? Existing demo artifacts under that directory will be removed.`,
|
||||
initialValue: false,
|
||||
});
|
||||
return confirmed ? { action: 'use', projectDir, reset: true } : { action: 'cancel' };
|
||||
}
|
||||
|
||||
return { action: 'use', projectDir, reset: false };
|
||||
}
|
||||
|
||||
export async function resolveFullCredentialDecision(options: {
|
||||
needsAnthropicKey: boolean;
|
||||
inputMode?: KloDemoInputMode;
|
||||
io: DemoInteractiveIo;
|
||||
env: NodeJS.ProcessEnv;
|
||||
prompts?: DemoPromptAdapter;
|
||||
}): Promise<FullCredentialDecision> {
|
||||
const env = cloneEnv(options.env);
|
||||
if (!options.needsAnthropicKey || env.ANTHROPIC_API_KEY) {
|
||||
return { action: 'full', env };
|
||||
}
|
||||
|
||||
if (!isInteractive(options.inputMode, options.io)) {
|
||||
return { action: 'full', env };
|
||||
}
|
||||
|
||||
const prompts = options.prompts ?? createClackDemoPromptAdapter();
|
||||
const choice = await prompts.select({
|
||||
message: 'Anthropic credentials are missing for the full demo',
|
||||
options: [
|
||||
{ value: 'process_key', label: 'Enter key for this process only' },
|
||||
{ value: 'seeded', label: 'Run pre-seeded demo without LLM' },
|
||||
{ value: 'replay', label: 'Run packaged replay' },
|
||||
{ value: 'cancel', label: 'Cancel' },
|
||||
],
|
||||
});
|
||||
|
||||
if (choice === 'cancel') {
|
||||
prompts.cancel('Demo cancelled.');
|
||||
return { action: 'cancel' };
|
||||
}
|
||||
|
||||
if (choice === 'seeded' || choice === 'replay') {
|
||||
return { action: 'run-mode', mode: choice };
|
||||
}
|
||||
|
||||
const key = await prompts.password({ message: 'ANTHROPIC_API_KEY' });
|
||||
return { action: 'full', env: { ...env, ANTHROPIC_API_KEY: key } };
|
||||
}
|
||||
137
packages/cli/src/demo-metrics.test.ts
Normal file
137
packages/cli/src/demo-metrics.test.ts
Normal file
|
|
@ -0,0 +1,137 @@
|
|||
import type { MemoryFlowEvent, MemoryFlowReplayInput } from '@klo/context/ingest/memory-flow';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
buildDemoMetrics,
|
||||
formatCost,
|
||||
formatDuration,
|
||||
formatEta,
|
||||
formatTokens,
|
||||
formatTokensPerSec,
|
||||
progressBar,
|
||||
} from './demo-metrics.js';
|
||||
|
||||
function snapshot(events: MemoryFlowEvent[], overrides: Partial<MemoryFlowReplayInput> = {}): MemoryFlowReplayInput {
|
||||
return {
|
||||
runId: 'run-1',
|
||||
connectionId: 'orbit_demo',
|
||||
adapter: 'live-database',
|
||||
status: 'running',
|
||||
sourceDir: null,
|
||||
syncId: 'sync-1',
|
||||
errors: [],
|
||||
events,
|
||||
plannedWorkUnits: [],
|
||||
details: { actions: [], provenance: [], transcripts: [] },
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe('buildDemoMetrics', () => {
|
||||
it('estimates elapsed, agent steps, tool calls, and cost from event stream', () => {
|
||||
const start = Date.UTC(2026, 0, 1, 0, 0, 0);
|
||||
const input = snapshot(
|
||||
[
|
||||
{ type: 'source_acquired', adapter: 'live-database', trigger: 'demo_full', fileCount: 5, emittedAt: new Date(start).toISOString() },
|
||||
{ type: 'work_unit_started', unitKey: 'orders', skills: [], stepBudget: 40, emittedAt: new Date(start + 1000).toISOString() },
|
||||
{ type: 'work_unit_step', unitKey: 'orders', stepIndex: 6, stepBudget: 40, emittedAt: new Date(start + 6000).toISOString() },
|
||||
],
|
||||
{
|
||||
plannedWorkUnits: [
|
||||
{ unitKey: 'orders', rawFiles: [], peerFileCount: 0, dependencyCount: 0 },
|
||||
{ unitKey: 'customers', rawFiles: [], peerFileCount: 0, dependencyCount: 0 },
|
||||
],
|
||||
details: {
|
||||
actions: [],
|
||||
provenance: [],
|
||||
transcripts: [{ unitKey: 'orders', path: '/tmp/orders.jsonl', toolCallCount: 3, errorCount: 0, toolNames: ['x'] }],
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
const metrics = buildDemoMetrics(input, { now: () => start + 10_000 });
|
||||
|
||||
expect(metrics.elapsedMs).toBe(10_000);
|
||||
expect(metrics.agentSteps).toBe(6);
|
||||
expect(metrics.agentStepBudget).toBe(40);
|
||||
expect(metrics.toolCalls).toBe(3);
|
||||
expect(metrics.workUnitsTotal).toBe(2);
|
||||
expect(metrics.estimatedTokens).toBeGreaterThan(0);
|
||||
expect(metrics.estimatedCostUsd).toBeGreaterThan(0);
|
||||
expect(metrics.isCostEstimated).toBe(true);
|
||||
});
|
||||
|
||||
it('returns null ETA before the first work unit completes', () => {
|
||||
const input = snapshot([{ type: 'source_acquired', adapter: 'live-database', trigger: 'x', fileCount: 1 }]);
|
||||
const metrics = buildDemoMetrics(input, { now: () => Date.now() });
|
||||
expect(metrics.etaMs).toBeNull();
|
||||
});
|
||||
|
||||
it('extrapolates ETA from completed/total ratio when at least one unit finishes', () => {
|
||||
const start = Date.UTC(2026, 0, 1);
|
||||
const input = snapshot(
|
||||
[
|
||||
{ type: 'source_acquired', adapter: 'a', trigger: 't', fileCount: 1, emittedAt: new Date(start).toISOString() },
|
||||
{ type: 'work_unit_started', unitKey: 'a', skills: [], stepBudget: 10, emittedAt: new Date(start + 1000).toISOString() },
|
||||
{ type: 'work_unit_finished', unitKey: 'a', status: 'success', emittedAt: new Date(start + 5000).toISOString() },
|
||||
],
|
||||
{
|
||||
plannedWorkUnits: [
|
||||
{ unitKey: 'a', rawFiles: [], peerFileCount: 0, dependencyCount: 0 },
|
||||
{ unitKey: 'b', rawFiles: [], peerFileCount: 0, dependencyCount: 0 },
|
||||
{ unitKey: 'c', rawFiles: [], peerFileCount: 0, dependencyCount: 0 },
|
||||
],
|
||||
},
|
||||
);
|
||||
|
||||
const metrics = buildDemoMetrics(input, { now: () => start + 6_000 });
|
||||
expect(metrics.etaMs).toBe(12_000);
|
||||
});
|
||||
|
||||
it('reports ETA=0 when the run is finished', () => {
|
||||
const input = snapshot([], { status: 'done' });
|
||||
const metrics = buildDemoMetrics(input, { now: () => Date.now() });
|
||||
expect(metrics.etaMs).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('format helpers', () => {
|
||||
it('formats duration in s/m/h cascades', () => {
|
||||
expect(formatDuration(5_000)).toBe('5s');
|
||||
expect(formatDuration(95_000)).toBe('1m35s');
|
||||
expect(formatDuration(3_700_000)).toBe('1h01m');
|
||||
expect(formatDuration(-1)).toBe('--');
|
||||
});
|
||||
|
||||
it('formats ETA as estimating before any data and as duration once running', () => {
|
||||
expect(formatEta(null, 'running')).toBe('estimating...');
|
||||
expect(formatEta(8_000, 'running')).toBe('8s');
|
||||
expect(formatEta(8_000, 'done')).toBe('done');
|
||||
});
|
||||
|
||||
it('formats cost with sub-cent guard', () => {
|
||||
expect(formatCost(0)).toBe('$0.000');
|
||||
expect(formatCost(0.0005)).toBe('<$0.001');
|
||||
expect(formatCost(0.012)).toBe('$0.012');
|
||||
expect(formatCost(2.5)).toBe('$2.50');
|
||||
});
|
||||
|
||||
it('formats token counts with K/M abbreviations', () => {
|
||||
expect(formatTokens(0)).toBe('0');
|
||||
expect(formatTokens(450)).toBe('450');
|
||||
expect(formatTokens(2_300)).toBe('2.3K');
|
||||
expect(formatTokens(1_500_000)).toBe('1.50M');
|
||||
});
|
||||
|
||||
it('formats tokens per second', () => {
|
||||
expect(formatTokensPerSec(0)).toBe('0/s');
|
||||
expect(formatTokensPerSec(450)).toBe('450/s');
|
||||
expect(formatTokensPerSec(2300)).toBe('2.3K/s');
|
||||
});
|
||||
|
||||
it('renders a deterministic progress bar with hash and dash characters', () => {
|
||||
expect(progressBar(0, 10)).toBe('----------');
|
||||
expect(progressBar(0.5, 10)).toBe('#####-----');
|
||||
expect(progressBar(1, 10)).toBe('##########');
|
||||
expect(progressBar(1.4, 10)).toBe('##########');
|
||||
});
|
||||
});
|
||||
174
packages/cli/src/demo-metrics.ts
Normal file
174
packages/cli/src/demo-metrics.ts
Normal file
|
|
@ -0,0 +1,174 @@
|
|||
import type { MemoryFlowEvent, MemoryFlowReplayInput } from '@klo/context/ingest/memory-flow';
|
||||
|
||||
const DEFAULT_INPUT_TOKENS_PER_STEP = 4500;
|
||||
const DEFAULT_OUTPUT_TOKENS_PER_STEP = 700;
|
||||
const DEFAULT_INPUT_PRICE_PER_MTOK_USD = 3;
|
||||
const DEFAULT_OUTPUT_PRICE_PER_MTOK_USD = 15;
|
||||
|
||||
interface DemoMetricsTuning {
|
||||
inputTokensPerStep?: number;
|
||||
outputTokensPerStep?: number;
|
||||
inputPricePerMTokUsd?: number;
|
||||
outputPricePerMTokUsd?: number;
|
||||
}
|
||||
|
||||
interface DemoMetricsSnapshot {
|
||||
elapsedMs: number;
|
||||
etaMs: number | null;
|
||||
agentSteps: number;
|
||||
agentStepBudget: number;
|
||||
toolCalls: number;
|
||||
workUnitsStarted: number;
|
||||
workUnitsFinished: number;
|
||||
workUnitsTotal: number;
|
||||
estimatedInputTokens: number;
|
||||
estimatedOutputTokens: number;
|
||||
estimatedTokens: number;
|
||||
estimatedCostUsd: number;
|
||||
tokensPerSec: number;
|
||||
status: MemoryFlowReplayInput['status'];
|
||||
isCostEstimated: boolean;
|
||||
}
|
||||
|
||||
function eventsOf<T extends MemoryFlowEvent['type']>(
|
||||
events: MemoryFlowEvent[],
|
||||
type: T,
|
||||
): Array<Extract<MemoryFlowEvent, { type: T }>> {
|
||||
return events.filter((event): event is Extract<MemoryFlowEvent, { type: T }> => event.type === type);
|
||||
}
|
||||
|
||||
function maxAgentStep(events: MemoryFlowEvent[]): { step: number; budget: number } {
|
||||
const steps = eventsOf(events, 'work_unit_step');
|
||||
const started = eventsOf(events, 'work_unit_started');
|
||||
const stepIndex = steps.reduce((max, event) => Math.max(max, event.stepIndex), 0);
|
||||
const stepBudget = Math.max(
|
||||
0,
|
||||
...steps.map((event) => event.stepBudget),
|
||||
...started.map((event) => event.stepBudget),
|
||||
);
|
||||
return { step: stepIndex, budget: stepBudget };
|
||||
}
|
||||
|
||||
function totalToolCalls(input: MemoryFlowReplayInput): number {
|
||||
return input.details.transcripts.reduce((total, transcript) => total + transcript.toolCallCount, 0);
|
||||
}
|
||||
|
||||
function workUnitProgress(input: MemoryFlowReplayInput): { started: number; finished: number; total: number } {
|
||||
const started = eventsOf(input.events, 'work_unit_started').length;
|
||||
const finished = eventsOf(input.events, 'work_unit_finished').length;
|
||||
const planned = input.plannedWorkUnits.length;
|
||||
const planEvent = eventsOf(input.events, 'chunks_planned').at(-1);
|
||||
const total = planned || planEvent?.workUnitCount || started || finished || 0;
|
||||
return { started, finished, total };
|
||||
}
|
||||
|
||||
function elapsedMsFromEvents(events: MemoryFlowEvent[], nowMs: number): number {
|
||||
const stamped = events
|
||||
.map((event) => (event.emittedAt ? Date.parse(event.emittedAt) : Number.NaN))
|
||||
.filter((value) => Number.isFinite(value));
|
||||
if (stamped.length === 0) return 0;
|
||||
const first = Math.min(...stamped);
|
||||
return Math.max(0, nowMs - first);
|
||||
}
|
||||
|
||||
function estimateEtaMs(
|
||||
elapsedMs: number,
|
||||
finished: number,
|
||||
total: number,
|
||||
status: MemoryFlowReplayInput['status'],
|
||||
): number | null {
|
||||
if (status !== 'running') return 0;
|
||||
if (total === 0 || finished === 0 || elapsedMs === 0) return null;
|
||||
const perUnit = elapsedMs / finished;
|
||||
const remaining = Math.max(0, total - finished);
|
||||
return Math.round(perUnit * remaining);
|
||||
}
|
||||
|
||||
export function buildDemoMetrics(
|
||||
input: MemoryFlowReplayInput,
|
||||
options: { now?: () => number; tuning?: DemoMetricsTuning } = {},
|
||||
): DemoMetricsSnapshot {
|
||||
const tuning = options.tuning ?? {};
|
||||
const inputTokensPerStep = tuning.inputTokensPerStep ?? DEFAULT_INPUT_TOKENS_PER_STEP;
|
||||
const outputTokensPerStep = tuning.outputTokensPerStep ?? DEFAULT_OUTPUT_TOKENS_PER_STEP;
|
||||
const inputPrice = tuning.inputPricePerMTokUsd ?? DEFAULT_INPUT_PRICE_PER_MTOK_USD;
|
||||
const outputPrice = tuning.outputPricePerMTokUsd ?? DEFAULT_OUTPUT_PRICE_PER_MTOK_USD;
|
||||
const nowMs = (options.now ?? Date.now)();
|
||||
const elapsedMs = elapsedMsFromEvents(input.events, nowMs);
|
||||
|
||||
const { step, budget } = maxAgentStep(input.events);
|
||||
const toolCalls = totalToolCalls(input);
|
||||
const progress = workUnitProgress(input);
|
||||
const finishedCount = eventsOf(input.events, 'work_unit_finished').length;
|
||||
const stepDriver = Math.max(step, toolCalls, finishedCount * 4);
|
||||
|
||||
const inputTokens = stepDriver * inputTokensPerStep;
|
||||
const outputTokens = stepDriver * outputTokensPerStep;
|
||||
const totalTokens = inputTokens + outputTokens;
|
||||
const cost = (inputTokens / 1_000_000) * inputPrice + (outputTokens / 1_000_000) * outputPrice;
|
||||
|
||||
const elapsedSec = elapsedMs / 1000;
|
||||
const tokensPerSec = elapsedSec > 0 ? totalTokens / elapsedSec : 0;
|
||||
|
||||
return {
|
||||
elapsedMs,
|
||||
etaMs: estimateEtaMs(elapsedMs, progress.finished, progress.total, input.status),
|
||||
agentSteps: step,
|
||||
agentStepBudget: budget,
|
||||
toolCalls,
|
||||
workUnitsStarted: progress.started,
|
||||
workUnitsFinished: progress.finished,
|
||||
workUnitsTotal: progress.total,
|
||||
estimatedInputTokens: inputTokens,
|
||||
estimatedOutputTokens: outputTokens,
|
||||
estimatedTokens: totalTokens,
|
||||
estimatedCostUsd: cost,
|
||||
tokensPerSec,
|
||||
status: input.status,
|
||||
isCostEstimated: true,
|
||||
};
|
||||
}
|
||||
|
||||
export function formatDuration(ms: number): string {
|
||||
if (!Number.isFinite(ms) || ms < 0) return '--';
|
||||
const totalSec = Math.round(ms / 1000);
|
||||
if (totalSec < 60) return `${totalSec}s`;
|
||||
const min = Math.floor(totalSec / 60);
|
||||
const sec = totalSec % 60;
|
||||
if (min < 60) return `${min}m${sec.toString().padStart(2, '0')}s`;
|
||||
const hr = Math.floor(min / 60);
|
||||
return `${hr}h${(min % 60).toString().padStart(2, '0')}m`;
|
||||
}
|
||||
|
||||
export function formatEta(ms: number | null, status: MemoryFlowReplayInput['status']): string {
|
||||
if (status !== 'running') return 'done';
|
||||
if (ms === null) return 'estimating...';
|
||||
return formatDuration(ms);
|
||||
}
|
||||
|
||||
export function formatCost(usd: number): string {
|
||||
if (!Number.isFinite(usd) || usd <= 0) return '$0.000';
|
||||
if (usd < 0.001) return '<$0.001';
|
||||
if (usd < 1) return `$${usd.toFixed(3)}`;
|
||||
return `$${usd.toFixed(2)}`;
|
||||
}
|
||||
|
||||
export function formatTokens(n: number): string {
|
||||
if (!Number.isFinite(n) || n <= 0) return '0';
|
||||
if (n < 1000) return `${Math.round(n)}`;
|
||||
if (n < 1_000_000) return `${(n / 1000).toFixed(1)}K`;
|
||||
return `${(n / 1_000_000).toFixed(2)}M`;
|
||||
}
|
||||
|
||||
export function formatTokensPerSec(n: number): string {
|
||||
if (!Number.isFinite(n) || n <= 0) return '0/s';
|
||||
if (n < 1000) return `${Math.round(n)}/s`;
|
||||
return `${(n / 1000).toFixed(1)}K/s`;
|
||||
}
|
||||
|
||||
const PROGRESS_BAR_WIDTH = 12;
|
||||
export function progressBar(ratio: number, width: number = PROGRESS_BAR_WIDTH): string {
|
||||
const clamped = Math.max(0, Math.min(1, ratio));
|
||||
const filled = Math.round(clamped * width);
|
||||
return `${'#'.repeat(filled)}${'-'.repeat(Math.max(0, width - filled))}`;
|
||||
}
|
||||
228
packages/cli/src/demo-progress.test.ts
Normal file
228
packages/cli/src/demo-progress.test.ts
Normal file
|
|
@ -0,0 +1,228 @@
|
|||
import type { MemoryFlowEvent, MemoryFlowReplayInput } from '@klo/context/ingest/memory-flow';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { createPlainProgressEmitter, formatMemoryFlowEventLine } from './demo-progress.js';
|
||||
|
||||
function snapshot(events: MemoryFlowEvent[]): MemoryFlowReplayInput {
|
||||
return {
|
||||
runId: 'run-1',
|
||||
connectionId: 'orbit_demo',
|
||||
adapter: 'live-database',
|
||||
status: 'running',
|
||||
sourceDir: null,
|
||||
syncId: 'sync-1',
|
||||
errors: [],
|
||||
events,
|
||||
plannedWorkUnits: [],
|
||||
details: { actions: [], provenance: [], transcripts: [] },
|
||||
};
|
||||
}
|
||||
|
||||
describe('formatMemoryFlowEventLine', () => {
|
||||
it('formats source_acquired in plain English with adapter and file count', () => {
|
||||
expect(
|
||||
formatMemoryFlowEventLine({
|
||||
type: 'source_acquired',
|
||||
adapter: 'live-database',
|
||||
trigger: 'manual_resync',
|
||||
fileCount: 7,
|
||||
}),
|
||||
).toBe('[connect] Connected live-database - 7 database files (manual_resync)');
|
||||
});
|
||||
|
||||
it('formats diff_computed as a comma-separated breakdown', () => {
|
||||
expect(
|
||||
formatMemoryFlowEventLine({
|
||||
type: 'diff_computed',
|
||||
added: 3,
|
||||
modified: 1,
|
||||
deleted: 0,
|
||||
unchanged: 4,
|
||||
}),
|
||||
).toBe('[diff] Tables: +3 new, ~1 changed, =4 unchanged');
|
||||
});
|
||||
|
||||
it('formats diff_computed as "no changes" when every counter is zero', () => {
|
||||
expect(
|
||||
formatMemoryFlowEventLine({
|
||||
type: 'diff_computed',
|
||||
added: 0,
|
||||
modified: 0,
|
||||
deleted: 0,
|
||||
unchanged: 0,
|
||||
}),
|
||||
).toBe('[diff] Tables: no changes');
|
||||
});
|
||||
|
||||
it('formats chunks_planned without removals as a single readable sentence', () => {
|
||||
expect(
|
||||
formatMemoryFlowEventLine({
|
||||
type: 'chunks_planned',
|
||||
chunkCount: 7,
|
||||
workUnitCount: 5,
|
||||
evictionCount: 0,
|
||||
}),
|
||||
).toBe('[plan] Grouped 5 tables into 7 business areas');
|
||||
});
|
||||
|
||||
it('formats chunks_planned with removals when evictions are non-zero', () => {
|
||||
expect(
|
||||
formatMemoryFlowEventLine({
|
||||
type: 'chunks_planned',
|
||||
chunkCount: 7,
|
||||
workUnitCount: 5,
|
||||
evictionCount: 2,
|
||||
}),
|
||||
).toBe('[plan] Grouped 5 tables into 7 business areas (2 removals)');
|
||||
});
|
||||
|
||||
it('formats work_unit_started in human terms', () => {
|
||||
expect(
|
||||
formatMemoryFlowEventLine({
|
||||
type: 'work_unit_started',
|
||||
unitKey: 'revenue-policy',
|
||||
skills: ['sl_expert', 'wiki_writer'],
|
||||
stepBudget: 40,
|
||||
}),
|
||||
).toBe('[analyze] Reviewing "revenue-policy" - budget 40 agent steps');
|
||||
});
|
||||
|
||||
it('suppresses noisy work_unit_step events', () => {
|
||||
expect(
|
||||
formatMemoryFlowEventLine({
|
||||
type: 'work_unit_step',
|
||||
unitKey: 'revenue-policy',
|
||||
stepIndex: 3,
|
||||
stepBudget: 40,
|
||||
}),
|
||||
).toBeNull();
|
||||
});
|
||||
|
||||
it('formats candidate_action with friendly target and arrow', () => {
|
||||
expect(
|
||||
formatMemoryFlowEventLine({
|
||||
type: 'candidate_action',
|
||||
unitKey: 'revenue-policy',
|
||||
target: 'sl',
|
||||
action: 'created',
|
||||
key: 'warehouse.revenue',
|
||||
}),
|
||||
).toBe('[draft] revenue-policy -> semantic-layer: created warehouse.revenue');
|
||||
});
|
||||
|
||||
it('formats work_unit_finished with status-aware tag', () => {
|
||||
expect(
|
||||
formatMemoryFlowEventLine({
|
||||
type: 'work_unit_finished',
|
||||
unitKey: 'revenue-policy',
|
||||
status: 'success',
|
||||
}),
|
||||
).toBe('[done] revenue-policy reviewed');
|
||||
|
||||
expect(
|
||||
formatMemoryFlowEventLine({
|
||||
type: 'work_unit_finished',
|
||||
unitKey: 'revenue-policy',
|
||||
status: 'failed',
|
||||
reason: 'budget exhausted',
|
||||
}),
|
||||
).toBe('[fail] revenue-policy needs attention - budget exhausted');
|
||||
});
|
||||
|
||||
it('formats reconciliation_finished with friendly counter wording', () => {
|
||||
expect(
|
||||
formatMemoryFlowEventLine({
|
||||
type: 'reconciliation_finished',
|
||||
conflictCount: 0,
|
||||
fallbackCount: 0,
|
||||
}),
|
||||
).toBe('[validate] Reconciled drafts - no conflicts, nothing flagged for review');
|
||||
|
||||
expect(
|
||||
formatMemoryFlowEventLine({
|
||||
type: 'reconciliation_finished',
|
||||
conflictCount: 2,
|
||||
fallbackCount: 1,
|
||||
}),
|
||||
).toBe('[validate] Reconciled drafts - 2 conflicts, 1 item flagged for review');
|
||||
});
|
||||
|
||||
it('formats saved with optional shortened commit sha and pluralized memory count', () => {
|
||||
expect(
|
||||
formatMemoryFlowEventLine({
|
||||
type: 'saved',
|
||||
commitSha: 'abc1234567890', // pragma: allowlist secret
|
||||
wikiCount: 2,
|
||||
slCount: 5,
|
||||
}),
|
||||
).toBe('[memory] Saved 7 memories (2 wiki, 5 semantic-layer) - commit abc1234');
|
||||
|
||||
expect(
|
||||
formatMemoryFlowEventLine({
|
||||
type: 'saved',
|
||||
commitSha: null,
|
||||
wikiCount: 0,
|
||||
slCount: 1,
|
||||
}),
|
||||
).toBe('[memory] Saved 1 memory (0 wiki, 1 semantic-layer)');
|
||||
});
|
||||
|
||||
it('formats report_created with run id', () => {
|
||||
expect(
|
||||
formatMemoryFlowEventLine({
|
||||
type: 'report_created',
|
||||
runId: 'run-xyz',
|
||||
}),
|
||||
).toBe('[report] Run report ready: run-xyz');
|
||||
});
|
||||
});
|
||||
|
||||
describe('createPlainProgressEmitter', () => {
|
||||
it('writes one line per new event and never re-emits prior events', () => {
|
||||
const written: string[] = [];
|
||||
const io = {
|
||||
stdout: { write: (chunk: string) => written.push(chunk), isTTY: false },
|
||||
stderr: { write: () => undefined },
|
||||
};
|
||||
const emit = createPlainProgressEmitter(io);
|
||||
|
||||
emit(
|
||||
snapshot([
|
||||
{ type: 'source_acquired', adapter: 'live-database', trigger: 'manual_resync', fileCount: 7 },
|
||||
{ type: 'diff_computed', added: 0, modified: 0, deleted: 0, unchanged: 7 },
|
||||
]),
|
||||
);
|
||||
|
||||
emit(
|
||||
snapshot([
|
||||
{ type: 'source_acquired', adapter: 'live-database', trigger: 'manual_resync', fileCount: 7 },
|
||||
{ type: 'diff_computed', added: 0, modified: 0, deleted: 0, unchanged: 7 },
|
||||
{ type: 'work_unit_started', unitKey: 'revenue-policy', skills: ['sl_expert'], stepBudget: 40 },
|
||||
]),
|
||||
);
|
||||
|
||||
expect(written).toEqual([
|
||||
'[connect] Connected live-database - 7 database files (manual_resync)\n',
|
||||
'[diff] Tables: =7 unchanged\n',
|
||||
'[analyze] Reviewing "revenue-policy" - budget 40 agent steps\n',
|
||||
]);
|
||||
});
|
||||
|
||||
it('skips suppressed events without advancing visible output', () => {
|
||||
const written: string[] = [];
|
||||
const io = {
|
||||
stdout: { write: (chunk: string) => written.push(chunk), isTTY: false },
|
||||
stderr: { write: () => undefined },
|
||||
};
|
||||
const emit = createPlainProgressEmitter(io);
|
||||
|
||||
emit(
|
||||
snapshot([
|
||||
{ type: 'work_unit_step', unitKey: 'a', stepIndex: 1, stepBudget: 40 },
|
||||
{ type: 'work_unit_step', unitKey: 'a', stepIndex: 2, stepBudget: 40 },
|
||||
{ type: 'work_unit_finished', unitKey: 'a', status: 'success' },
|
||||
]),
|
||||
);
|
||||
|
||||
expect(written).toEqual(['[done] a reviewed\n']);
|
||||
});
|
||||
});
|
||||
77
packages/cli/src/demo-progress.ts
Normal file
77
packages/cli/src/demo-progress.ts
Normal file
|
|
@ -0,0 +1,77 @@
|
|||
import type { MemoryFlowEvent, MemoryFlowReplayInput } from '@klo/context/ingest/memory-flow';
|
||||
import type { KloDemoIo } from './demo.js';
|
||||
|
||||
function plural(n: number, one: string, many = `${one}s`): string {
|
||||
return `${n} ${n === 1 ? one : many}`;
|
||||
}
|
||||
|
||||
function formatDiff(added: number, modified: number, deleted: number, unchanged: number): string {
|
||||
const parts: string[] = [];
|
||||
if (added > 0) parts.push(`+${added} new`);
|
||||
if (modified > 0) parts.push(`~${modified} changed`);
|
||||
if (deleted > 0) parts.push(`-${deleted} removed`);
|
||||
if (unchanged > 0) parts.push(`=${unchanged} unchanged`);
|
||||
return parts.length > 0 ? parts.join(', ') : 'no changes';
|
||||
}
|
||||
|
||||
export function formatMemoryFlowEventLine(event: MemoryFlowEvent): string | null {
|
||||
switch (event.type) {
|
||||
case 'source_acquired':
|
||||
return `[connect] Connected ${event.adapter} - ${plural(event.fileCount, 'database file')} (${event.trigger})`;
|
||||
case 'scope_detected':
|
||||
return event.fingerprint
|
||||
? `[scope] Scope locked: ${event.fingerprint}`
|
||||
: '[scope] Reviewing the whole warehouse (no scope filter)';
|
||||
case 'raw_snapshot_written':
|
||||
return `[snapshot] Captured snapshot ${event.syncId} - ${plural(event.rawFileCount, 'file')}`;
|
||||
case 'diff_computed':
|
||||
return `[diff] Tables: ${formatDiff(event.added, event.modified, event.deleted, event.unchanged)}`;
|
||||
case 'chunks_planned':
|
||||
return event.evictionCount > 0
|
||||
? `[plan] Grouped ${plural(event.workUnitCount, 'table')} into ${plural(event.chunkCount, 'business area')} (${plural(event.evictionCount, 'removal')})`
|
||||
: `[plan] Grouped ${plural(event.workUnitCount, 'table')} into ${plural(event.chunkCount, 'business area')}`;
|
||||
case 'stage_skipped':
|
||||
return `[skip] ${event.stage} skipped: ${event.reason}`;
|
||||
case 'work_unit_started':
|
||||
return `[analyze] Reviewing "${event.unitKey}" - budget ${plural(event.stepBudget, 'agent step')}`;
|
||||
case 'work_unit_step':
|
||||
return null;
|
||||
case 'candidate_action': {
|
||||
const target = event.target === 'sl' ? 'semantic-layer' : 'wiki';
|
||||
return `[draft] ${event.unitKey} -> ${target}: ${event.action} ${event.key}`;
|
||||
}
|
||||
case 'work_unit_finished':
|
||||
if (event.status === 'success') {
|
||||
return `[done] ${event.unitKey} reviewed`;
|
||||
}
|
||||
return `[fail] ${event.unitKey} needs attention${event.reason ? ` - ${event.reason}` : ''}`;
|
||||
case 'reconciliation_finished': {
|
||||
const conflicts = event.conflictCount === 0 ? 'no conflicts' : plural(event.conflictCount, 'conflict');
|
||||
const fallbacks = event.fallbackCount === 0 ? 'nothing flagged for review' : `${plural(event.fallbackCount, 'item')} flagged for review`;
|
||||
return `[validate] Reconciled drafts - ${conflicts}, ${fallbacks}`;
|
||||
}
|
||||
case 'saved': {
|
||||
const total = event.wikiCount + event.slCount;
|
||||
const commit = event.commitSha ? ` - commit ${event.commitSha.slice(0, 7)}` : '';
|
||||
return `[memory] Saved ${plural(total, 'memory', 'memories')} (${event.wikiCount} wiki, ${event.slCount} semantic-layer)${commit}`;
|
||||
}
|
||||
case 'provenance_recorded':
|
||||
return `[trace] Recorded provenance for ${plural(event.rowCount, 'row')}`;
|
||||
case 'report_created':
|
||||
return `[report] Run report ready: ${event.runId}`;
|
||||
}
|
||||
}
|
||||
|
||||
export function createPlainProgressEmitter(io: KloDemoIo): (snapshot: MemoryFlowReplayInput) => void {
|
||||
let printed = 0;
|
||||
return (snapshot) => {
|
||||
while (printed < snapshot.events.length) {
|
||||
const event = snapshot.events[printed++];
|
||||
if (!event) continue;
|
||||
const line = formatMemoryFlowEventLine(event);
|
||||
if (line !== null) {
|
||||
io.stdout.write(`${line}\n`);
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
60
packages/cli/src/demo-replay-store.test.ts
Normal file
60
packages/cli/src/demo-replay-store.test.ts
Normal file
|
|
@ -0,0 +1,60 @@
|
|||
import { mkdtemp, readFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { type MemoryFlowReplayInput } from '@klo/context/ingest/memory-flow';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { DEMO_LATEST_REPLAY_FILE, loadLatestDemoReplay, writeDemoReplay } from './demo-replay-store.js';
|
||||
|
||||
function replay(overrides: Partial<MemoryFlowReplayInput> = {}): MemoryFlowReplayInput {
|
||||
return {
|
||||
metadata: {
|
||||
schemaVersion: 1,
|
||||
mode: 'full',
|
||||
origin: 'captured',
|
||||
timing: 'captured',
|
||||
capturedAt: '2026-05-01T10:00:03.000Z',
|
||||
sourceReportId: 'report-1',
|
||||
sourceReportPath: 'report-1',
|
||||
fallbackReason: null,
|
||||
},
|
||||
runId: 'run-1',
|
||||
connectionId: 'orbit_demo',
|
||||
adapter: 'live-database',
|
||||
status: 'done',
|
||||
sourceDir: null,
|
||||
syncId: 'sync-1',
|
||||
reportId: 'report-1',
|
||||
reportPath: 'report-1',
|
||||
errors: [],
|
||||
events: [{ type: 'report_created', runId: 'run-1', reportPath: 'report-1', emittedAt: '2026-05-01T10:00:03.000Z' }],
|
||||
plannedWorkUnits: [],
|
||||
details: { actions: [], provenance: [], transcripts: [] },
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe('demo replay store', () => {
|
||||
it('writes a versioned replay file and updates latest', async () => {
|
||||
const projectDir = await mkdtemp(join(tmpdir(), 'klo-demo-replay-store-'));
|
||||
|
||||
const saved = await writeDemoReplay(projectDir, replay(), { label: 'full' });
|
||||
|
||||
expect(saved.replayPath).toMatch(/replays[/\\]full-run-1.memory-flow.v1.json$/);
|
||||
expect(saved.latestReplayPath).toBe(join(projectDir, 'replays', DEMO_LATEST_REPLAY_FILE));
|
||||
expect(await loadLatestDemoReplay(projectDir)).toMatchObject({
|
||||
runId: 'run-1',
|
||||
metadata: { mode: 'full', origin: 'captured', timing: 'captured' },
|
||||
});
|
||||
|
||||
const wrapper = JSON.parse(await readFile(saved.latestReplayPath, 'utf-8')) as {
|
||||
memoryFlowReplaySchemaVersion?: number;
|
||||
};
|
||||
expect(wrapper.memoryFlowReplaySchemaVersion).toBe(1);
|
||||
});
|
||||
|
||||
it('returns null when no latest local replay exists', async () => {
|
||||
const projectDir = await mkdtemp(join(tmpdir(), 'klo-demo-replay-store-empty-'));
|
||||
|
||||
await expect(loadLatestDemoReplay(projectDir)).resolves.toBeNull();
|
||||
});
|
||||
});
|
||||
68
packages/cli/src/demo-replay-store.ts
Normal file
68
packages/cli/src/demo-replay-store.ts
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
import { constants as fsConstants } from 'node:fs';
|
||||
import { access, copyFile, mkdir, readFile, writeFile } from 'node:fs/promises';
|
||||
import { join, resolve } from 'node:path';
|
||||
import { parseMemoryFlowReplayInput, type MemoryFlowReplayInput } from '@klo/context/ingest/memory-flow';
|
||||
|
||||
interface StoredMemoryFlowReplayFile {
|
||||
memoryFlowReplaySchemaVersion: 1;
|
||||
replay: unknown;
|
||||
}
|
||||
|
||||
interface SavedDemoReplay {
|
||||
replayPath: string;
|
||||
latestReplayPath: string;
|
||||
}
|
||||
|
||||
export const DEMO_LATEST_REPLAY_FILE = 'latest.memory-flow.v1.json';
|
||||
|
||||
async function exists(path: string): Promise<boolean> {
|
||||
try {
|
||||
await access(path, fsConstants.F_OK);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function safeReplayName(value: string): string {
|
||||
return value.replace(/[^a-zA-Z0-9._-]+/g, '-').replace(/^-+|-+$/g, '') || 'replay';
|
||||
}
|
||||
|
||||
function demoReplayFileName(input: MemoryFlowReplayInput, label: string): string {
|
||||
return `${safeReplayName(label)}-${safeReplayName(input.runId)}.memory-flow.v1.json`;
|
||||
}
|
||||
|
||||
function wrapReplay(input: MemoryFlowReplayInput): StoredMemoryFlowReplayFile {
|
||||
return { memoryFlowReplaySchemaVersion: 1, replay: input };
|
||||
}
|
||||
|
||||
export async function loadDemoReplayFile(path: string): Promise<MemoryFlowReplayInput> {
|
||||
const parsed = JSON.parse(await readFile(path, 'utf-8')) as StoredMemoryFlowReplayFile;
|
||||
if (parsed.memoryFlowReplaySchemaVersion !== 1) {
|
||||
throw new Error(`Unsupported demo replay schema version in ${path}`);
|
||||
}
|
||||
return parseMemoryFlowReplayInput(parsed.replay);
|
||||
}
|
||||
|
||||
export async function loadLatestDemoReplay(projectDir: string): Promise<MemoryFlowReplayInput | null> {
|
||||
const latestPath = join(resolve(projectDir), 'replays', DEMO_LATEST_REPLAY_FILE);
|
||||
if (!(await exists(latestPath))) {
|
||||
return null;
|
||||
}
|
||||
return loadDemoReplayFile(latestPath);
|
||||
}
|
||||
|
||||
export async function writeDemoReplay(
|
||||
projectDir: string,
|
||||
input: MemoryFlowReplayInput,
|
||||
options: { label: 'full' | 'deterministic' | 'seeded' },
|
||||
): Promise<SavedDemoReplay> {
|
||||
const replayDir = join(resolve(projectDir), 'replays');
|
||||
await mkdir(replayDir, { recursive: true });
|
||||
const replayPath = join(replayDir, demoReplayFileName(input, options.label));
|
||||
const latestReplayPath = join(replayDir, DEMO_LATEST_REPLAY_FILE);
|
||||
const body = `${JSON.stringify(wrapReplay(input), null, 2)}\n`;
|
||||
await writeFile(replayPath, body, 'utf-8');
|
||||
await copyFile(replayPath, latestReplayPath);
|
||||
return { replayPath, latestReplayPath };
|
||||
}
|
||||
31
packages/cli/src/demo-scan.test.ts
Normal file
31
packages/cli/src/demo-scan.test.ts
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
import { rm } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterEach, describe, expect, it } from 'vitest';
|
||||
import { findLatestDemoScanReport, runDemoScan } from './demo-scan.js';
|
||||
|
||||
describe('demo scan helpers', () => {
|
||||
const projectDir = join(tmpdir(), `klo-demo-scan-${process.pid}`);
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(projectDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('runs the packaged SQLite demo scan and finds the latest scan report', async () => {
|
||||
const { result } = await runDemoScan({
|
||||
projectDir,
|
||||
jobId: 'demo-scan-test',
|
||||
now: () => new Date('2026-05-06T10:00:00.000Z'),
|
||||
});
|
||||
|
||||
expect(result.report).toMatchObject({
|
||||
connectionId: 'orbit_demo',
|
||||
driver: 'sqlite',
|
||||
runId: 'demo-scan-test',
|
||||
mode: 'structural',
|
||||
dryRun: false,
|
||||
});
|
||||
expect(result.report.artifactPaths.reportPath).toContain('raw-sources/orbit_demo/live-database/');
|
||||
await expect(findLatestDemoScanReport(projectDir)).resolves.toMatchObject({ runId: 'demo-scan-test' });
|
||||
});
|
||||
});
|
||||
223
packages/cli/src/demo-scan.ts
Normal file
223
packages/cli/src/demo-scan.ts
Normal file
|
|
@ -0,0 +1,223 @@
|
|||
import { getLocalIngestStatus, type IngestReportSnapshot, type MemoryFlowReplayInput } from '@klo/context/ingest';
|
||||
import { loadKloProject, type KloLocalProject } from '@klo/context/project';
|
||||
import { runLocalScan, type KloScanReport, type LocalScanRunResult } from '@klo/context/scan';
|
||||
import { DEMO_ADAPTER, DEMO_CONNECTION_ID, DEMO_FULL_JOB_ID, ensureDemoProject } from './demo-assets.js';
|
||||
import { loadLatestDemoReplay } from './demo-replay-store.js';
|
||||
import { createKloCliLocalIngestAdapters } from './local-adapters.js';
|
||||
|
||||
interface DemoScanOptions {
|
||||
projectDir: string;
|
||||
jobId?: string;
|
||||
now?: () => Date;
|
||||
runLocalScan?: typeof runLocalScan;
|
||||
}
|
||||
|
||||
interface DemoScanResult {
|
||||
project: KloLocalProject;
|
||||
result: LocalScanRunResult;
|
||||
}
|
||||
|
||||
interface DemoInspectSummary {
|
||||
projectDir: string;
|
||||
scanReport: KloScanReport | null;
|
||||
fullReport: IngestReportSnapshot | null;
|
||||
semanticLayerFileCount: number;
|
||||
knowledgeFileCount: number;
|
||||
replayFileCount: number;
|
||||
latestReplay: MemoryFlowReplayInput | null;
|
||||
}
|
||||
|
||||
interface DemoInspectDeps {
|
||||
findFullReport?: (project: KloLocalProject) => Promise<IngestReportSnapshot | null>;
|
||||
}
|
||||
|
||||
async function ensureDemoProjectForReuse(projectDir: string): Promise<void> {
|
||||
await ensureDemoProject({ projectDir, force: false }).catch((error) => {
|
||||
if (error instanceof Error && error.message.includes('Demo project already exists')) {
|
||||
return;
|
||||
}
|
||||
throw error;
|
||||
});
|
||||
}
|
||||
|
||||
async function loadReadyDemoProject(projectDir: string): Promise<KloLocalProject> {
|
||||
try {
|
||||
return await loadKloProject({ projectDir });
|
||||
} catch (error) {
|
||||
const reason = error instanceof Error ? error.message : String(error);
|
||||
throw new Error(
|
||||
`Demo project is not ready at ${projectDir}: ${reason}. Run klo setup demo init --project-dir ${projectDir} --force --no-input to recreate it.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function reportDiff(report: KloScanReport): string {
|
||||
return `+${report.diffSummary.tablesAdded}/~${report.diffSummary.tablesModified}/-${report.diffSummary.tablesDeleted}/=${report.diffSummary.tablesUnchanged}`;
|
||||
}
|
||||
|
||||
function jsonReport(raw: string, path: string): KloScanReport {
|
||||
try {
|
||||
return JSON.parse(raw) as KloScanReport;
|
||||
} catch (error) {
|
||||
const reason = error instanceof Error ? error.message : String(error);
|
||||
throw new Error(`Invalid demo scan report at ${path}: ${reason}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function countFiles(project: KloLocalProject, root: string, predicate: (path: string) => boolean): Promise<number> {
|
||||
const { files } = await project.fileStore.listFiles(root, true);
|
||||
return files.filter(predicate).length;
|
||||
}
|
||||
|
||||
async function findFullDemoReport(project: KloLocalProject): Promise<IngestReportSnapshot | null> {
|
||||
return getLocalIngestStatus(project, DEMO_FULL_JOB_ID);
|
||||
}
|
||||
|
||||
function savedCounts(report: IngestReportSnapshot): { wikiCount: number; slCount: number } {
|
||||
const actions = report.body.workUnits.flatMap((workUnit) => workUnit.actions);
|
||||
return {
|
||||
wikiCount: actions.filter((action) => action.target === 'wiki').length,
|
||||
slCount: actions.filter((action) => action.target === 'sl').length,
|
||||
};
|
||||
}
|
||||
|
||||
export async function runDemoScan(options: DemoScanOptions): Promise<DemoScanResult> {
|
||||
await ensureDemoProjectForReuse(options.projectDir);
|
||||
const project = await loadReadyDemoProject(options.projectDir);
|
||||
const executeScan = options.runLocalScan ?? runLocalScan;
|
||||
const result = await executeScan({
|
||||
project,
|
||||
connectionId: DEMO_CONNECTION_ID,
|
||||
mode: 'structural',
|
||||
trigger: 'cli',
|
||||
jobId: options.jobId ?? 'demo-scan',
|
||||
now: options.now,
|
||||
adapters: createKloCliLocalIngestAdapters(project),
|
||||
});
|
||||
|
||||
return { project, result };
|
||||
}
|
||||
|
||||
export async function findLatestDemoScanReport(projectDir: string): Promise<KloScanReport | null> {
|
||||
const project = await loadReadyDemoProject(projectDir);
|
||||
const root = `raw-sources/${DEMO_CONNECTION_ID}/${DEMO_ADAPTER}`;
|
||||
const { files } = await project.fileStore.listFiles(root, true);
|
||||
const latest = files
|
||||
.filter((path) => path.endsWith('/scan-report.json'))
|
||||
.sort()
|
||||
.at(-1);
|
||||
if (!latest) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const reportPath = `${root}/${latest}`;
|
||||
const report = await project.fileStore.readFile(reportPath);
|
||||
return jsonReport(report.content, reportPath);
|
||||
}
|
||||
|
||||
export async function inspectDemoProject(
|
||||
projectDir: string,
|
||||
projectOverride?: KloLocalProject,
|
||||
deps: DemoInspectDeps = {},
|
||||
): Promise<DemoInspectSummary> {
|
||||
const project = projectOverride ?? (await loadReadyDemoProject(projectDir));
|
||||
const scanReport = await findLatestDemoScanReport(project.projectDir);
|
||||
const fullReport = await (deps.findFullReport ?? findFullDemoReport)(project);
|
||||
const semanticLayerFileCount = await countFiles(
|
||||
project,
|
||||
`semantic-layer/${DEMO_CONNECTION_ID}`,
|
||||
(path) => path.endsWith('.yaml') || path.endsWith('.yml'),
|
||||
);
|
||||
const knowledgeFileCount = await countFiles(project, 'knowledge', (path) => path.endsWith('.md'));
|
||||
const replayFileCount = await countFiles(project, 'replays', (path) => path.endsWith('.json'));
|
||||
const latestReplay = await loadLatestDemoReplay(project.projectDir);
|
||||
|
||||
return {
|
||||
projectDir: project.projectDir,
|
||||
scanReport,
|
||||
fullReport,
|
||||
semanticLayerFileCount,
|
||||
knowledgeFileCount,
|
||||
replayFileCount,
|
||||
latestReplay,
|
||||
};
|
||||
}
|
||||
|
||||
export function formatDemoScanSummary(report: KloScanReport): string {
|
||||
return [
|
||||
'Demo scan: done',
|
||||
`Connection: ${report.connectionId}`,
|
||||
`Driver: ${report.driver}`,
|
||||
`Mode: ${report.mode}`,
|
||||
`Tables: ${reportDiff(report)}`,
|
||||
`Semantic-layer artifacts: ${report.artifactPaths.manifestShards.length}`,
|
||||
`Report: ${report.artifactPaths.reportPath ?? 'none'}`,
|
||||
'Next: klo setup demo inspect',
|
||||
' Shows the files and semantic-layer draft created from the database scan.',
|
||||
'',
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
function replayLine(replay: MemoryFlowReplayInput | null): string {
|
||||
if (!replay?.metadata) {
|
||||
return 'Latest replay: packaged demo replay';
|
||||
}
|
||||
return `Latest replay: ${replay.metadata.mode} (${replay.metadata.origin}, ${replay.metadata.timing})`;
|
||||
}
|
||||
|
||||
export function formatDemoInspect(summary: DemoInspectSummary): string {
|
||||
const report = summary.scanReport;
|
||||
const fullReport = summary.fullReport;
|
||||
const fullCounts = fullReport ? savedCounts(fullReport) : null;
|
||||
const scanLines = report
|
||||
? [
|
||||
'Scan artifacts: yes',
|
||||
`Connection: ${report.connectionId}`,
|
||||
`Driver: ${report.driver}`,
|
||||
`Tables: ${reportDiff(report)}`,
|
||||
`Report: ${report.artifactPaths.reportPath ?? 'none'}`,
|
||||
]
|
||||
: ['Scan artifacts: none'];
|
||||
|
||||
const memoryLines = fullReport
|
||||
? [
|
||||
'Memory synthesis: ran',
|
||||
`Full report: ${fullReport.id}`,
|
||||
`Full run: ${fullReport.runId}`,
|
||||
`Saved memory: ${fullCounts?.wikiCount ?? 0} wiki, ${fullCounts?.slCount ?? 0} semantic layer`,
|
||||
`Provenance rows: ${fullReport.body.provenanceRows.length}`,
|
||||
]
|
||||
: [report ? 'Memory synthesis: full mode not run' : 'Memory synthesis: not run'];
|
||||
const next = fullReport
|
||||
? [
|
||||
`Next: klo ingest watch ${fullReport.runId} --project-dir ${summary.projectDir}`,
|
||||
' Opens the captured run timeline and lets you inspect what happened.',
|
||||
'Next: klo setup demo replay',
|
||||
' Replays the same visual story without calling the LLM again.',
|
||||
]
|
||||
: report
|
||||
? [
|
||||
'Next: klo setup demo --mode full',
|
||||
' Runs the full AI-backed pass with your LLM provider.',
|
||||
'Next: klo setup demo replay',
|
||||
' Replays the packaged visual story without calling the LLM.',
|
||||
]
|
||||
: [
|
||||
'Next: klo setup demo --no-input',
|
||||
' Runs the pre-seeded demo without calling the LLM.',
|
||||
'Next: klo setup demo --mode full',
|
||||
' Runs the full AI-backed pass with your LLM provider.',
|
||||
];
|
||||
|
||||
return [
|
||||
`Demo project: ${summary.projectDir}`,
|
||||
...scanLines,
|
||||
`Semantic-layer files: ${summary.semanticLayerFileCount}`,
|
||||
`Knowledge files: ${summary.knowledgeFileCount}`,
|
||||
`Replay files: ${summary.replayFileCount}`,
|
||||
replayLine(summary.latestReplay),
|
||||
...memoryLines,
|
||||
...next,
|
||||
'',
|
||||
].join('\n');
|
||||
}
|
||||
123
packages/cli/src/demo-seeded-inspect.test.ts
Normal file
123
packages/cli/src/demo-seeded-inspect.test.ts
Normal file
|
|
@ -0,0 +1,123 @@
|
|||
import { access, readFile, rm } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterEach, describe, expect, it } from 'vitest';
|
||||
import { runDemoSeeded } from './demo-seeded.js';
|
||||
import { formatSeededInspect, inspectSeededProject } from './demo-seeded-inspect.js';
|
||||
import { KLO_NEXT_STEP_COMMANDS } from './next-steps.js';
|
||||
|
||||
describe('seeded demo inspect contract', () => {
|
||||
const projectDir = join(tmpdir(), `klo-demo-seeded-inspect-${process.pid}`);
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(projectDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('reports the PRD source inventory, generated outputs, status, metadata, and next commands', async () => {
|
||||
await runDemoSeeded({ projectDir });
|
||||
const inspect = await inspectSeededProject(projectDir);
|
||||
|
||||
expect(inspect).toMatchObject({
|
||||
projectDir,
|
||||
mode: 'seeded',
|
||||
status: { status: 'ready', missing: [] },
|
||||
modeMetadata: {
|
||||
mode: 'seeded',
|
||||
source: 'packaged demo project',
|
||||
generatedContext: 'prebuilt from bundled assets',
|
||||
llmCalls: 'none',
|
||||
origin: 'packaged',
|
||||
timing: 'prebuilt',
|
||||
sourceReportId: 'demo-seeded-report',
|
||||
sourceReportPath: 'reports/seeded-demo-report.json',
|
||||
},
|
||||
sourceBundle: {
|
||||
warehouse: {
|
||||
label: 'Warehouse',
|
||||
path: 'demo.db',
|
||||
tableCount: 8,
|
||||
totalRows: 11234,
|
||||
rowCounts: {
|
||||
accounts: 210,
|
||||
arr_movements: 720,
|
||||
contracts: 320,
|
||||
invoices: 3000,
|
||||
plans: 4,
|
||||
purchase_requests: 5200,
|
||||
support_tickets: 520,
|
||||
users: 1260,
|
||||
},
|
||||
},
|
||||
dbt: { label: 'dbt', path: 'raw-sources/dbt', modelCount: 3, sourceTableCount: 8 },
|
||||
bi: { label: 'BI', path: 'raw-sources/bi', exploreCount: 5, dashboardCount: 2 },
|
||||
notion: { label: 'Notion', path: 'raw-sources/notion', pageCount: 8 },
|
||||
},
|
||||
generatedOutputs: {
|
||||
semanticLayer: { path: 'semantic-layer/orbit_demo', manifestSourceCount: 6, fileCount: 6 },
|
||||
knowledge: { path: 'knowledge/global', manifestPageCount: 10, fileCount: 10 },
|
||||
links: { path: 'links/provenance.json', manifestLinkCount: 23, linkCount: 23 },
|
||||
reports: { primaryPath: 'reports/seeded-demo-report.json', fileCount: 1 },
|
||||
replays: { primaryPath: 'replays/replay.memory-flow.v1.json', latestPath: 'replays/latest.memory-flow.v1.json' },
|
||||
},
|
||||
nextCommands: KLO_NEXT_STEP_COMMANDS,
|
||||
});
|
||||
|
||||
expect(inspect.generatedOutputs.replays.fileCount).toBeGreaterThanOrEqual(3);
|
||||
await expect(access(join(projectDir, inspect.generatedOutputs.reports.primaryPath))).resolves.toBeUndefined();
|
||||
await expect(access(join(projectDir, inspect.generatedOutputs.replays.primaryPath))).resolves.toBeUndefined();
|
||||
await expect(access(join(projectDir, inspect.generatedOutputs.replays.latestPath))).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it('formats seeded inspect from the normalized contract', async () => {
|
||||
await runDemoSeeded({ projectDir });
|
||||
const output = formatSeededInspect(await inspectSeededProject(projectDir));
|
||||
|
||||
expect(output).toContain(`Demo project: ${projectDir}`);
|
||||
expect(output).toContain('Status: ready');
|
||||
expect(output).toContain('Mode: seeded (pre-seeded demo project)');
|
||||
expect(output).toContain('Source: packaged demo project');
|
||||
expect(output).toContain('Generated context: prebuilt from bundled assets');
|
||||
expect(output).toContain('LLM calls: none');
|
||||
expect(output).toContain('Warehouse: 8 tables, 11,234 rows');
|
||||
expect(output).toContain('Rows: accounts 210, arr_movements 720, contracts 320, invoices 3000');
|
||||
expect(output).toContain('dbt: 3 models, 8 source tables');
|
||||
expect(output).toContain('BI: 5 explores, 2 dashboards');
|
||||
expect(output).toContain('Notion: 8 pages');
|
||||
expect(output).toContain('Semantic-layer sources: 6 manifest, 6 files');
|
||||
expect(output).toContain('Knowledge pages: 10 manifest, 10 files');
|
||||
expect(output).toContain('Evidence links: 23 manifest, 23 links');
|
||||
expect(output).toContain('Report: reports/seeded-demo-report.json');
|
||||
expect(output).toContain('Replay: replays/replay.memory-flow.v1.json');
|
||||
expect(output).toContain('Latest replay: seeded (packaged, prebuilt)');
|
||||
expect(output).toContain(' $ klo agent tools --json');
|
||||
expect(output).toContain(' $ klo agent context --json');
|
||||
expect(output).toContain(' $ klo serve --mcp stdio --user-id local');
|
||||
expect(output.indexOf('klo agent tools --json')).toBeLessThan(
|
||||
output.indexOf('klo serve --mcp stdio --user-id local'),
|
||||
);
|
||||
expect(output).not.toContain('klo ask');
|
||||
expect(output).not.toContain('deterministic mode');
|
||||
});
|
||||
|
||||
it('reports missing seeded paths without reading stale counts as ready', async () => {
|
||||
await runDemoSeeded({ projectDir });
|
||||
await rm(join(projectDir, 'links', 'provenance.json'));
|
||||
|
||||
const inspect = await inspectSeededProject(projectDir);
|
||||
|
||||
expect(inspect.status).toEqual({ status: 'corrupt', missing: ['links/provenance.json'] });
|
||||
expect(formatSeededInspect(inspect)).toContain('Status: corrupt');
|
||||
expect(formatSeededInspect(inspect)).toContain('Missing: links/provenance.json');
|
||||
});
|
||||
|
||||
it('keeps provenance link counts tied to the project file', async () => {
|
||||
await runDemoSeeded({ projectDir });
|
||||
|
||||
const inspect = await inspectSeededProject(projectDir);
|
||||
const raw = await readFile(join(projectDir, 'links', 'provenance.json'), 'utf-8');
|
||||
const links = JSON.parse(raw) as unknown[];
|
||||
|
||||
expect(inspect.generatedOutputs.links.linkCount).toBe(links.length);
|
||||
expect(inspect.generatedOutputs.links.linkCount).toBe(23);
|
||||
});
|
||||
});
|
||||
299
packages/cli/src/demo-seeded-inspect.ts
Normal file
299
packages/cli/src/demo-seeded-inspect.ts
Normal file
|
|
@ -0,0 +1,299 @@
|
|||
import { constants as fsConstants } from 'node:fs';
|
||||
import { access, readFile, readdir } from 'node:fs/promises';
|
||||
import { join, resolve } from 'node:path';
|
||||
import type { MemoryFlowReplayInput } from '@klo/context/ingest/memory-flow';
|
||||
import { loadPackagedDemoReplay } from './demo-assets.js';
|
||||
import { DEMO_LATEST_REPLAY_FILE, loadLatestDemoReplay } from './demo-replay-store.js';
|
||||
import { KLO_NEXT_STEP_COMMANDS, KLO_NEXT_STEP_COMMAND_WIDTH } from './next-steps.js';
|
||||
|
||||
type SeededInspectReadiness = 'missing' | 'ready' | 'corrupt';
|
||||
|
||||
export interface DemoSeededManifest {
|
||||
demoAssetSchemaVersion: number;
|
||||
name: string;
|
||||
displayName: string;
|
||||
mode: string;
|
||||
source?: string;
|
||||
sources: {
|
||||
warehouse: { label: string; path?: string; tables: number; rowCounts: Record<string, number> };
|
||||
dbt: { label: string; path?: string; models: number; sourceTables: number };
|
||||
bi: { label: string; path?: string; explores: number; dashboards: number };
|
||||
notion: { label: string; path?: string; pages: number };
|
||||
};
|
||||
generated: {
|
||||
semanticLayer: { path?: string; sourceCount: number };
|
||||
knowledge: { path?: string; pageCount: number };
|
||||
links: { path?: string; linkCount: number };
|
||||
};
|
||||
}
|
||||
|
||||
export interface SeededInspectSummary {
|
||||
projectDir: string;
|
||||
mode: 'seeded';
|
||||
manifest: DemoSeededManifest;
|
||||
status: { status: SeededInspectReadiness; missing: string[] };
|
||||
sourceBundle: {
|
||||
warehouse: {
|
||||
label: string;
|
||||
path: string;
|
||||
tableCount: number;
|
||||
rowCounts: Record<string, number>;
|
||||
totalRows: number;
|
||||
};
|
||||
dbt: { label: string; path: string; modelCount: number; sourceTableCount: number };
|
||||
bi: { label: string; path: string; exploreCount: number; dashboardCount: number };
|
||||
notion: { label: string; path: string; pageCount: number };
|
||||
};
|
||||
generatedOutputs: {
|
||||
semanticLayer: { path: string; manifestSourceCount: number; fileCount: number };
|
||||
knowledge: { path: string; manifestPageCount: number; fileCount: number };
|
||||
links: { path: string; manifestLinkCount: number; linkCount: number };
|
||||
reports: { primaryPath: string; fileCount: number };
|
||||
replays: { primaryPath: string; latestPath: string; fileCount: number };
|
||||
};
|
||||
modeMetadata: {
|
||||
mode: 'seeded';
|
||||
source: 'packaged demo project';
|
||||
generatedContext: 'prebuilt from bundled assets';
|
||||
llmCalls: 'none';
|
||||
origin: string;
|
||||
timing: string;
|
||||
sourceReportId: string | null;
|
||||
sourceReportPath: string | null;
|
||||
};
|
||||
nextCommands: Array<{ command: string; description: string }>;
|
||||
latestReplay: MemoryFlowReplayInput | null;
|
||||
}
|
||||
|
||||
const REQUIRED_SEEDED_PROJECT_PATHS = [
|
||||
'klo.yaml',
|
||||
'demo.db',
|
||||
'state.sqlite',
|
||||
'manifest.json',
|
||||
join('replays', 'replay.memory-flow.v1.json'),
|
||||
join('raw-sources', 'warehouse', 'accounts.csv'),
|
||||
join('raw-sources', 'dbt', 'schema.yml'),
|
||||
join('raw-sources', 'bi', 'revenue_exec.dashboard.lookml'),
|
||||
join('raw-sources', 'notion', 'revenue-reporting-policy.md'),
|
||||
join('semantic-layer', 'orbit_demo', 'accounts.yaml'),
|
||||
join('knowledge', 'global', 'arr-contract-first.md'),
|
||||
join('links', 'provenance.json'),
|
||||
join('reports', 'seeded-demo-report.json'),
|
||||
] as const;
|
||||
|
||||
async function exists(path: string): Promise<boolean> {
|
||||
try {
|
||||
await access(path, fsConstants.F_OK);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function loadSeededManifest(projectDir: string): Promise<DemoSeededManifest> {
|
||||
const raw = await readFile(join(projectDir, 'manifest.json'), 'utf-8');
|
||||
return JSON.parse(raw) as DemoSeededManifest;
|
||||
}
|
||||
|
||||
async function listFilesInDir(dir: string, ext?: string): Promise<string[]> {
|
||||
try {
|
||||
const entries = await readdir(dir, { recursive: true });
|
||||
return entries
|
||||
.filter((entry): entry is string => typeof entry === 'string')
|
||||
.filter((entry) => !ext || entry.endsWith(ext))
|
||||
.sort();
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async function inspectSeededProjectStatus(projectDir: string): Promise<{ status: SeededInspectReadiness; missing: string[] }> {
|
||||
const missing: string[] = [];
|
||||
for (const relativePath of REQUIRED_SEEDED_PROJECT_PATHS) {
|
||||
if (!(await exists(join(projectDir, relativePath)))) {
|
||||
missing.push(relativePath);
|
||||
}
|
||||
}
|
||||
|
||||
if (missing.length === REQUIRED_SEEDED_PROJECT_PATHS.length) {
|
||||
return { status: 'missing', missing };
|
||||
}
|
||||
if (missing.length > 0) {
|
||||
return { status: 'corrupt', missing };
|
||||
}
|
||||
return { status: 'ready', missing: [] };
|
||||
}
|
||||
|
||||
async function loadLinksCount(projectDir: string): Promise<number> {
|
||||
try {
|
||||
const raw = await readFile(join(projectDir, 'links', 'provenance.json'), 'utf-8');
|
||||
const links = JSON.parse(raw) as unknown[];
|
||||
return links.length;
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
async function loadSeededReplay(projectDir: string): Promise<MemoryFlowReplayInput | null> {
|
||||
const latest = await loadLatestDemoReplay(projectDir);
|
||||
if (latest) {
|
||||
return latest;
|
||||
}
|
||||
|
||||
try {
|
||||
return await loadPackagedDemoReplay();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function sourceBundleFromManifest(manifest: DemoSeededManifest): SeededInspectSummary['sourceBundle'] {
|
||||
const warehouse = manifest.sources.warehouse;
|
||||
const rowCounts = Object.fromEntries(Object.entries(warehouse.rowCounts).sort(([a], [b]) => a.localeCompare(b)));
|
||||
const totalRows = Object.values(rowCounts).reduce((total, count) => total + count, 0);
|
||||
|
||||
return {
|
||||
warehouse: {
|
||||
label: warehouse.label,
|
||||
path: warehouse.path ?? 'demo.db',
|
||||
tableCount: warehouse.tables,
|
||||
rowCounts,
|
||||
totalRows,
|
||||
},
|
||||
dbt: {
|
||||
label: manifest.sources.dbt.label,
|
||||
path: manifest.sources.dbt.path ?? 'raw-sources/dbt',
|
||||
modelCount: manifest.sources.dbt.models,
|
||||
sourceTableCount: manifest.sources.dbt.sourceTables,
|
||||
},
|
||||
bi: {
|
||||
label: manifest.sources.bi.label,
|
||||
path: manifest.sources.bi.path ?? 'raw-sources/bi',
|
||||
exploreCount: manifest.sources.bi.explores,
|
||||
dashboardCount: manifest.sources.bi.dashboards,
|
||||
},
|
||||
notion: {
|
||||
label: manifest.sources.notion.label,
|
||||
path: manifest.sources.notion.path ?? 'raw-sources/notion',
|
||||
pageCount: manifest.sources.notion.pages,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function nextCommands(): SeededInspectSummary['nextCommands'] {
|
||||
return [...KLO_NEXT_STEP_COMMANDS];
|
||||
}
|
||||
|
||||
function modeMetadataFromReplay(replay: MemoryFlowReplayInput | null): SeededInspectSummary['modeMetadata'] {
|
||||
return {
|
||||
mode: 'seeded',
|
||||
source: 'packaged demo project',
|
||||
generatedContext: 'prebuilt from bundled assets',
|
||||
llmCalls: 'none',
|
||||
origin: replay?.metadata?.origin ?? 'packaged',
|
||||
timing: replay?.metadata?.timing ?? 'prebuilt',
|
||||
sourceReportId: replay?.metadata?.sourceReportId ?? 'demo-seeded-report',
|
||||
sourceReportPath: replay?.metadata?.sourceReportPath ?? 'reports/seeded-demo-report.json',
|
||||
};
|
||||
}
|
||||
|
||||
export async function inspectSeededProject(projectDir: string): Promise<SeededInspectSummary> {
|
||||
const root = resolve(projectDir);
|
||||
const manifest = await loadSeededManifest(root);
|
||||
const latestReplay = await loadSeededReplay(root);
|
||||
const semanticLayerPath = manifest.generated.semanticLayer.path ?? 'semantic-layer/orbit_demo';
|
||||
const knowledgePath = manifest.generated.knowledge.path ?? 'knowledge/global';
|
||||
const linksPath = join(manifest.generated.links.path ?? 'links', 'provenance.json');
|
||||
const reportFiles = await listFilesInDir(join(root, 'reports'), '.json');
|
||||
const replayFiles = await listFilesInDir(join(root, 'replays'), '.json');
|
||||
|
||||
return {
|
||||
projectDir: root,
|
||||
mode: 'seeded',
|
||||
manifest,
|
||||
status: await inspectSeededProjectStatus(root),
|
||||
sourceBundle: sourceBundleFromManifest(manifest),
|
||||
generatedOutputs: {
|
||||
semanticLayer: {
|
||||
path: semanticLayerPath,
|
||||
manifestSourceCount: manifest.generated.semanticLayer.sourceCount,
|
||||
fileCount: (await listFilesInDir(join(root, semanticLayerPath), '.yaml')).length,
|
||||
},
|
||||
knowledge: {
|
||||
path: knowledgePath,
|
||||
manifestPageCount: manifest.generated.knowledge.pageCount,
|
||||
fileCount: (await listFilesInDir(join(root, knowledgePath), '.md')).length,
|
||||
},
|
||||
links: {
|
||||
path: linksPath,
|
||||
manifestLinkCount: manifest.generated.links.linkCount,
|
||||
linkCount: await loadLinksCount(root),
|
||||
},
|
||||
reports: {
|
||||
primaryPath: reportFiles[0] ? join('reports', reportFiles[0]) : 'reports/seeded-demo-report.json',
|
||||
fileCount: reportFiles.length,
|
||||
},
|
||||
replays: {
|
||||
primaryPath: join('replays', 'replay.memory-flow.v1.json'),
|
||||
latestPath: join('replays', DEMO_LATEST_REPLAY_FILE),
|
||||
fileCount: replayFiles.length,
|
||||
},
|
||||
},
|
||||
modeMetadata: modeMetadataFromReplay(latestReplay),
|
||||
nextCommands: nextCommands(),
|
||||
latestReplay,
|
||||
};
|
||||
}
|
||||
|
||||
function rowCountPreview(rowCounts: Record<string, number>): string {
|
||||
return Object.entries(rowCounts)
|
||||
.map(([name, count]) => `${name} ${count}`)
|
||||
.join(', ');
|
||||
}
|
||||
|
||||
function replayLine(summary: SeededInspectSummary): string {
|
||||
const metadata = summary.latestReplay?.metadata ?? summary.modeMetadata;
|
||||
return `Latest replay: ${metadata.mode} (${metadata.origin}, ${metadata.timing})`;
|
||||
}
|
||||
|
||||
export function formatSeededInspect(summary: SeededInspectSummary): string {
|
||||
const source = summary.sourceBundle;
|
||||
const generated = summary.generatedOutputs;
|
||||
const lines = [`Demo project: ${summary.projectDir}`, `Status: ${summary.status.status}`];
|
||||
|
||||
if (summary.status.missing.length > 0) {
|
||||
lines.push(`Missing: ${summary.status.missing.join(', ')}`);
|
||||
}
|
||||
|
||||
lines.push(
|
||||
`Mode: seeded (pre-seeded demo project)`,
|
||||
`Source: ${summary.modeMetadata.source}`,
|
||||
`Generated context: ${summary.modeMetadata.generatedContext}`,
|
||||
`LLM calls: ${summary.modeMetadata.llmCalls}`,
|
||||
'',
|
||||
'Source bundle:',
|
||||
` Warehouse: ${source.warehouse.tableCount} tables, ${source.warehouse.totalRows.toLocaleString()} rows`,
|
||||
` Rows: ${rowCountPreview(source.warehouse.rowCounts)}`,
|
||||
` dbt: ${source.dbt.modelCount} models, ${source.dbt.sourceTableCount} source tables`,
|
||||
` BI: ${source.bi.exploreCount} explores, ${source.bi.dashboardCount} dashboards`,
|
||||
` Notion: ${source.notion.pageCount} pages`,
|
||||
'',
|
||||
'Generated context:',
|
||||
` Semantic-layer sources: ${generated.semanticLayer.manifestSourceCount} manifest, ${generated.semanticLayer.fileCount} files`,
|
||||
` Knowledge pages: ${generated.knowledge.manifestPageCount} manifest, ${generated.knowledge.fileCount} files`,
|
||||
` Evidence links: ${generated.links.manifestLinkCount} manifest, ${generated.links.linkCount} links`,
|
||||
'',
|
||||
`Report: ${generated.reports.primaryPath}`,
|
||||
`Replay: ${generated.replays.primaryPath}`,
|
||||
replayLine(summary),
|
||||
'',
|
||||
'What to do next:',
|
||||
);
|
||||
|
||||
for (const command of summary.nextCommands) {
|
||||
lines.push(` $ ${command.command.padEnd(KLO_NEXT_STEP_COMMAND_WIDTH)} ${command.description}`);
|
||||
}
|
||||
|
||||
lines.push('', `Your KLO project files are at: ${summary.projectDir}`, '');
|
||||
return lines.join('\n');
|
||||
}
|
||||
117
packages/cli/src/demo-seeded.test.ts
Normal file
117
packages/cli/src/demo-seeded.test.ts
Normal file
|
|
@ -0,0 +1,117 @@
|
|||
import { access, readFile, rm } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterEach, describe, expect, it } from 'vitest';
|
||||
import { ensureSeededDemoProject } from './demo-assets.js';
|
||||
import { runDemoSeeded } from './demo-seeded.js';
|
||||
|
||||
describe('demo seeded mode', () => {
|
||||
const projectDir = join(tmpdir(), `klo-demo-seeded-${process.pid}`);
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(projectDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('hydrates a complete seeded project with all asset directories', async () => {
|
||||
const result = await ensureSeededDemoProject({ projectDir, force: false });
|
||||
|
||||
expect(result.projectDir).toBe(projectDir);
|
||||
await expect(access(join(projectDir, 'demo.db'))).resolves.toBeUndefined();
|
||||
await expect(access(join(projectDir, 'klo.yaml'))).resolves.toBeUndefined();
|
||||
await expect(access(join(projectDir, 'manifest.json'))).resolves.toBeUndefined();
|
||||
await expect(access(join(projectDir, 'semantic-layer/orbit_demo/accounts.yaml'))).resolves.toBeUndefined();
|
||||
await expect(access(join(projectDir, 'knowledge/global/arr-contract-first.md'))).resolves.toBeUndefined();
|
||||
await expect(access(join(projectDir, 'raw-sources/dbt/schema.yml'))).resolves.toBeUndefined();
|
||||
await expect(access(join(projectDir, 'raw-sources/bi/revenue_exec.dashboard.lookml'))).resolves.toBeUndefined();
|
||||
await expect(access(join(projectDir, 'raw-sources/notion/revenue-reporting-policy.md'))).resolves.toBeUndefined();
|
||||
await expect(access(join(projectDir, 'links/provenance.json'))).resolves.toBeUndefined();
|
||||
await expect(access(join(projectDir, 'reports/seeded-demo-report.json'))).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it('does not load or call any LLM provider in seeded mode', async () => {
|
||||
const result = await runDemoSeeded({ projectDir });
|
||||
|
||||
expect(result.replay.metadata?.mode).toBe('seeded');
|
||||
expect(result.replay.metadata?.timing).toBe('prebuilt');
|
||||
expect(result.inspect.mode).toBe('seeded');
|
||||
|
||||
const config = await readFile(join(projectDir, 'klo.yaml'), 'utf-8');
|
||||
expect(config).toContain('api_key: env:ANTHROPIC_API_KEY');
|
||||
expect(config).not.toContain('sk-ant-');
|
||||
});
|
||||
|
||||
it('creates the project under /tmp by default', async () => {
|
||||
const result = await runDemoSeeded({ projectDir });
|
||||
expect(result.projectDir).toBe(projectDir);
|
||||
});
|
||||
|
||||
it('replay metadata identifies mode honestly', async () => {
|
||||
const result = await runDemoSeeded({ projectDir });
|
||||
|
||||
expect(result.replay.metadata).toMatchObject({
|
||||
mode: 'seeded',
|
||||
origin: 'packaged',
|
||||
timing: 'prebuilt',
|
||||
});
|
||||
expect(result.replay.runId).toBe('demo-seeded-orbit');
|
||||
});
|
||||
|
||||
it('packaged seeded replay is honest and shows every source family', async () => {
|
||||
const result = await runDemoSeeded({ projectDir });
|
||||
const sourceEvents = result.replay.events.filter((event) => event.type === 'source_acquired');
|
||||
const adapters = sourceEvents.map((event) => event.adapter).sort();
|
||||
|
||||
expect(result.replay.metadata).toMatchObject({
|
||||
mode: 'seeded',
|
||||
origin: 'packaged',
|
||||
timing: 'prebuilt',
|
||||
sourceReportPath: 'reports/seeded-demo-report.json',
|
||||
});
|
||||
expect(adapters).toEqual(['dbt_descriptions', 'live-database', 'looker', 'notion']);
|
||||
expect(result.replay.events).not.toContainEqual(
|
||||
expect.objectContaining({ type: 'stage_skipped', reason: expect.stringContaining('deterministic') }),
|
||||
);
|
||||
expect(JSON.stringify(result.replay)).not.toContain('LLM ran');
|
||||
});
|
||||
|
||||
it('seeded animation shows all demo source families', async () => {
|
||||
const result = await runDemoSeeded({ projectDir });
|
||||
const adapters = result.replay.events
|
||||
.filter((e) => e.type === 'source_acquired')
|
||||
.map((e) => (e as { adapter: string }).adapter);
|
||||
|
||||
expect(adapters).toContain('live-database');
|
||||
expect(adapters).toContain('dbt_descriptions');
|
||||
expect(adapters).toContain('looker');
|
||||
expect(adapters).toContain('notion');
|
||||
});
|
||||
|
||||
it('SL YAML validates correctly', async () => {
|
||||
await ensureSeededDemoProject({ projectDir, force: false });
|
||||
const slYaml = await readFile(join(projectDir, 'semantic-layer/orbit_demo/accounts.yaml'), 'utf-8');
|
||||
expect(slYaml).toContain('name: accounts');
|
||||
expect(slYaml).toContain('grain:');
|
||||
expect(slYaml).toContain('columns:');
|
||||
expect(slYaml).toContain('measures:');
|
||||
expect(slYaml).toContain('joins:');
|
||||
});
|
||||
|
||||
it('wiki pages have valid frontmatter', async () => {
|
||||
await ensureSeededDemoProject({ projectDir, force: false });
|
||||
const wiki = await readFile(join(projectDir, 'knowledge/global/arr-contract-first.md'), 'utf-8');
|
||||
expect(wiki).toContain('---');
|
||||
expect(wiki).toContain('summary:');
|
||||
expect(wiki).toContain('tags:');
|
||||
expect(wiki).toContain('sl_refs:');
|
||||
expect(wiki).toContain('usage_mode: auto');
|
||||
});
|
||||
|
||||
it('links are searchable through provenance file', async () => {
|
||||
await ensureSeededDemoProject({ projectDir, force: false });
|
||||
const raw = await readFile(join(projectDir, 'links/provenance.json'), 'utf-8');
|
||||
const links = JSON.parse(raw) as Array<{ id: string; artifactKind: string }>;
|
||||
expect(links.length).toBe(23);
|
||||
expect(links.some((l) => l.artifactKind === 'wiki')).toBe(true);
|
||||
expect(links.some((l) => l.artifactKind === 'sl')).toBe(true);
|
||||
});
|
||||
});
|
||||
41
packages/cli/src/demo-seeded.ts
Normal file
41
packages/cli/src/demo-seeded.ts
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
import type { MemoryFlowReplayInput } from '@klo/context/ingest/memory-flow';
|
||||
import {
|
||||
ensureSeededDemoProject,
|
||||
loadPackagedDemoReplay,
|
||||
} from './demo-assets.js';
|
||||
import { writeDemoReplay } from './demo-replay-store.js';
|
||||
import { inspectSeededProject, type SeededInspectSummary } from './demo-seeded-inspect.js';
|
||||
|
||||
export {
|
||||
formatSeededInspect,
|
||||
inspectSeededProject,
|
||||
type DemoSeededManifest,
|
||||
type SeededInspectSummary,
|
||||
} from './demo-seeded-inspect.js';
|
||||
|
||||
export interface DemoSeededResult {
|
||||
projectDir: string;
|
||||
replay: MemoryFlowReplayInput;
|
||||
inspect: SeededInspectSummary;
|
||||
}
|
||||
|
||||
export async function runDemoSeeded(options: {
|
||||
projectDir: string;
|
||||
}): Promise<DemoSeededResult> {
|
||||
const result = await ensureSeededDemoProject({ projectDir: options.projectDir, force: false });
|
||||
|
||||
const replay = await loadPackagedDemoReplay();
|
||||
const replayWithDir: MemoryFlowReplayInput = {
|
||||
...replay,
|
||||
sourceDir: result.projectDir,
|
||||
};
|
||||
|
||||
await writeDemoReplay(result.projectDir, replayWithDir, { label: 'seeded' });
|
||||
const inspect = await inspectSeededProject(result.projectDir);
|
||||
|
||||
return {
|
||||
projectDir: result.projectDir,
|
||||
replay: replayWithDir,
|
||||
inspect,
|
||||
};
|
||||
}
|
||||
751
packages/cli/src/demo.test.ts
Normal file
751
packages/cli/src/demo.test.ts
Normal file
|
|
@ -0,0 +1,751 @@
|
|||
import { mkdtemp, readFile, rm } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import type { IngestReportSnapshot, MemoryFlowReplayInput } from '@klo/context/ingest';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { runKloDemo } from './demo.js';
|
||||
import { DEMO_FULL_JOB_ID, defaultDemoProjectDir, ensureDemoProject } from './demo-assets.js';
|
||||
import type { DemoFullResult } from './demo-full.js';
|
||||
import { createTestDemoPromptAdapter } from './demo-interaction.js';
|
||||
import type { renderMemoryFlowTui } from './memory-flow-tui.js';
|
||||
import { KLO_NEXT_STEP_COMMANDS } from './next-steps.js';
|
||||
import { resetVizFallbackWarningsForTest } from './viz-fallback.js';
|
||||
|
||||
function makeIo(options: { isTTY?: boolean; columns?: number; rawMode?: boolean } = {}) {
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
return {
|
||||
io: {
|
||||
stdin: {
|
||||
isTTY: options.isTTY ?? false,
|
||||
...(options.rawMode === false ? {} : { setRawMode: vi.fn() }),
|
||||
},
|
||||
stdout: {
|
||||
isTTY: options.isTTY ?? false,
|
||||
columns: options.columns ?? 140,
|
||||
write: (chunk: string) => {
|
||||
stdout += chunk;
|
||||
},
|
||||
},
|
||||
stderr: {
|
||||
write: (chunk: string) => {
|
||||
stderr += chunk;
|
||||
},
|
||||
},
|
||||
},
|
||||
stdout: () => stdout,
|
||||
stderr: () => stderr,
|
||||
};
|
||||
}
|
||||
|
||||
function fakeFullResult(projectDir: string): DemoFullResult {
|
||||
const report: IngestReportSnapshot = {
|
||||
id: 'report-full',
|
||||
runId: 'run-full',
|
||||
jobId: DEMO_FULL_JOB_ID,
|
||||
connectionId: 'orbit_demo',
|
||||
sourceKey: 'live-database',
|
||||
createdAt: '2026-05-01T00:00:00.000Z',
|
||||
body: {
|
||||
syncId: 'sync-full',
|
||||
diffSummary: { added: 7, modified: 0, deleted: 0, unchanged: 0 },
|
||||
commitSha: null,
|
||||
workUnits: [
|
||||
{
|
||||
unitKey: 'accounts',
|
||||
rawFiles: ['accounts.schema.json'],
|
||||
status: 'success',
|
||||
actions: [
|
||||
{ target: 'wiki', type: 'created', key: 'knowledge/accounts.md', detail: 'account lifecycle context' },
|
||||
{ target: 'sl', type: 'created', key: 'orbit_demo.accounts', detail: 'accounts semantic source' },
|
||||
],
|
||||
touchedSlSources: [{ connectionId: 'orbit_demo', sourceName: 'orbit_demo.accounts' }],
|
||||
},
|
||||
],
|
||||
failedWorkUnits: [],
|
||||
reconciliationSkipped: false,
|
||||
conflictsResolved: [],
|
||||
evictionsApplied: [],
|
||||
unmappedFallbacks: [],
|
||||
evictionInputs: [],
|
||||
unresolvedCards: [],
|
||||
supersededBy: null,
|
||||
overrideOf: null,
|
||||
provenanceRows: [
|
||||
{
|
||||
rawPath: 'accounts.schema.json',
|
||||
artifactKind: 'wiki',
|
||||
artifactKey: 'knowledge/accounts.md',
|
||||
actionType: 'wiki_written',
|
||||
},
|
||||
],
|
||||
toolTranscripts: [],
|
||||
},
|
||||
};
|
||||
|
||||
return {
|
||||
project: { projectDir } as never,
|
||||
scan: { report: { runId: 'scan-run' } } as never,
|
||||
ingest: { result: { ok: true }, report } as never,
|
||||
report,
|
||||
replay: {
|
||||
runId: 'run-full',
|
||||
connectionId: 'orbit_demo',
|
||||
adapter: 'live-database',
|
||||
status: 'done',
|
||||
sourceDir: `${projectDir}/raw-sources/orbit_demo/live-database/sync-full`,
|
||||
syncId: 'sync-full',
|
||||
errors: [],
|
||||
events: [
|
||||
{ type: 'source_acquired', adapter: 'live-database', trigger: 'demo_full', fileCount: 7 },
|
||||
{ type: 'saved', commitSha: null, wikiCount: 1, slCount: 1 },
|
||||
{ type: 'provenance_recorded', rowCount: 1 },
|
||||
{ type: 'report_created', runId: 'run-full', reportPath: 'report-full' },
|
||||
],
|
||||
plannedWorkUnits: [],
|
||||
details: { actions: [], provenance: [], transcripts: [] },
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe('runKloDemo', () => {
|
||||
let tempDir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
resetVizFallbackWarningsForTest();
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'klo-demo-command-'));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(tempDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('initializes the demo project', async () => {
|
||||
const io = makeIo();
|
||||
await expect(
|
||||
runKloDemo({ command: 'init', projectDir: tempDir, force: false, inputMode: 'disabled' }, io.io),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(io.stdout()).toContain(`Demo project: ${tempDir}`);
|
||||
expect(io.stdout()).toContain('Config:');
|
||||
expect(io.stdout()).toContain('Replay:');
|
||||
expect(io.stderr()).toBe('');
|
||||
});
|
||||
|
||||
it('renders the packaged replay in no-input viz mode', async () => {
|
||||
const io = makeIo({ isTTY: true });
|
||||
await expect(
|
||||
runKloDemo(
|
||||
{ command: 'replay', projectDir: tempDir, outputMode: 'viz', inputMode: 'disabled' },
|
||||
io.io,
|
||||
{ env: { ...process.env, TERM: 'xterm-256color' } },
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(io.stdout()).toContain('KLO memory flow Warehouse + dbt + BI + Docs done');
|
||||
expect(io.stdout()).toContain('Saved 16 memories');
|
||||
expect(io.stderr()).toBe('');
|
||||
});
|
||||
|
||||
it('routes interactive packaged replay viz through the stored TUI renderer', async () => {
|
||||
const io = makeIo({ isTTY: true });
|
||||
const renderStoredMemoryFlow = vi.fn<typeof renderMemoryFlowTui>(async () => true);
|
||||
|
||||
await expect(
|
||||
runKloDemo(
|
||||
{ command: 'replay', projectDir: tempDir, outputMode: 'viz' },
|
||||
io.io,
|
||||
{ env: { ...process.env, TERM: 'xterm-256color' }, renderStoredMemoryFlow },
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(renderStoredMemoryFlow).toHaveBeenCalledTimes(1);
|
||||
expect(renderStoredMemoryFlow.mock.calls[0]?.[0]).toMatchObject({
|
||||
runId: 'demo-seeded-orbit',
|
||||
connectionId: 'orbit_demo',
|
||||
adapter: 'live-database',
|
||||
});
|
||||
expect(renderStoredMemoryFlow.mock.calls[0]?.[2]).toEqual({ speedMultiplier: 0.125 });
|
||||
expect(io.stdout()).toContain('KLO finished ingesting your data');
|
||||
expect(io.stderr()).toBe('');
|
||||
});
|
||||
|
||||
it('routes interactive seeded demo viz through the stored TUI renderer at eighth speed', async () => {
|
||||
const io = makeIo({ isTTY: true });
|
||||
const renderStoredMemoryFlow = vi.fn<typeof renderMemoryFlowTui>(async () => true);
|
||||
|
||||
await expect(
|
||||
runKloDemo(
|
||||
{ command: 'seeded', projectDir: tempDir, outputMode: 'viz' },
|
||||
io.io,
|
||||
{ env: { ...process.env, TERM: 'xterm-256color' }, renderStoredMemoryFlow },
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(renderStoredMemoryFlow).toHaveBeenCalledTimes(1);
|
||||
expect(renderStoredMemoryFlow.mock.calls[0]?.[2]).toEqual({ speedMultiplier: 0.125 });
|
||||
expect(io.stdout()).toContain('KLO finished ingesting your data');
|
||||
expect(io.stderr()).toBe('');
|
||||
});
|
||||
|
||||
it('falls back to plain replay output when interactive replay viz lacks stdin raw mode', async () => {
|
||||
const io = makeIo({ isTTY: true, rawMode: false });
|
||||
const renderStoredMemoryFlow = vi.fn(async (_input: MemoryFlowReplayInput, _io: unknown) => true);
|
||||
|
||||
await expect(
|
||||
runKloDemo(
|
||||
{ command: 'replay', projectDir: tempDir, outputMode: 'viz' },
|
||||
io.io,
|
||||
{ env: { ...process.env, TERM: 'xterm-256color' }, renderStoredMemoryFlow },
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(renderStoredMemoryFlow).not.toHaveBeenCalled();
|
||||
expect(io.stdout()).toContain('Memory-flow summary: done');
|
||||
expect(io.stdout()).toContain('Connection: orbit_demo');
|
||||
expect(io.stdout()).toContain('klo sl list');
|
||||
expect(io.stdout()).toContain('klo wiki list');
|
||||
expect(io.stdout()).toContain('klo serve --mcp stdio --user-id local');
|
||||
expect(io.stdout()).not.toContain('KLO memory flow');
|
||||
expect(io.stderr()).toContain(
|
||||
'Visualization requested but stdin raw mode is unavailable; printing plain output.',
|
||||
);
|
||||
});
|
||||
|
||||
it('degrades default visual demo replay to a plain memory-flow summary when stdout is redirected', async () => {
|
||||
const testIo = makeIo({ isTTY: false });
|
||||
|
||||
await expect(
|
||||
runKloDemo({ command: 'replay', projectDir: tempDir, outputMode: 'viz', inputMode: 'disabled' }, testIo.io),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(testIo.stdout()).toContain('Memory-flow summary: done');
|
||||
expect(testIo.stdout()).toContain('Connection: orbit_demo');
|
||||
expect(testIo.stdout()).toContain('klo sl list');
|
||||
expect(testIo.stdout()).toContain('klo wiki list');
|
||||
expect(testIo.stdout()).toContain('klo serve --mcp stdio --user-id local');
|
||||
expect(testIo.stdout()).not.toContain('KLO memory flow');
|
||||
expect(testIo.stderr()).toContain(
|
||||
'Visualization requested but stdout is not an interactive terminal; printing plain output.',
|
||||
);
|
||||
});
|
||||
|
||||
it('prints JSON replay output when requested', async () => {
|
||||
const io = makeIo();
|
||||
await expect(
|
||||
runKloDemo({ command: 'replay', projectDir: tempDir, outputMode: 'json', inputMode: 'disabled' }, io.io),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(JSON.parse(io.stdout())).toMatchObject({ runId: 'demo-seeded-orbit', connectionId: 'orbit_demo' });
|
||||
expect(io.stderr()).toBe('');
|
||||
});
|
||||
|
||||
it('runs the packaged SQLite demo scan', async () => {
|
||||
const io = makeIo();
|
||||
await expect(runKloDemo({ command: 'scan', projectDir: tempDir, inputMode: 'disabled' }, io.io)).resolves.toBe(0);
|
||||
|
||||
expect(io.stdout()).toContain('Demo scan: done');
|
||||
expect(io.stdout()).toContain('Connection: orbit_demo');
|
||||
expect(io.stdout()).toContain('Driver: sqlite');
|
||||
expect(io.stdout()).toContain('Report: raw-sources/orbit_demo/live-database/');
|
||||
expect(io.stderr()).toBe('');
|
||||
});
|
||||
|
||||
it('runs seeded mode with pre-seeded assets and inspect summary', async () => {
|
||||
const io = makeIo({ isTTY: true });
|
||||
await expect(
|
||||
runKloDemo(
|
||||
{ command: 'seeded', projectDir: tempDir, outputMode: 'plain', inputMode: 'disabled' },
|
||||
io.io,
|
||||
{ env: { ...process.env, TERM: 'xterm-256color' } },
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(io.stdout()).toContain('Mode: seeded');
|
||||
expect(io.stdout()).toContain('LLM calls: none');
|
||||
expect(io.stdout()).toContain('Semantic-layer sources:');
|
||||
expect(io.stdout()).toContain('Knowledge pages:');
|
||||
expect(io.stderr()).toBe('');
|
||||
});
|
||||
|
||||
it('uses seeded mode as the default demo and creates a temp project when no project-dir is supplied', async () => {
|
||||
const io = makeIo();
|
||||
|
||||
await expect(
|
||||
runKloDemo(
|
||||
{ command: 'seeded', projectDir: defaultDemoProjectDir(), outputMode: 'plain', inputMode: 'disabled' },
|
||||
io.io,
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(io.stdout()).toContain('Mode: seeded');
|
||||
expect(io.stdout()).toContain('Source: packaged demo project');
|
||||
expect(io.stdout()).toContain('Generated context: prebuilt from bundled assets');
|
||||
expect(io.stdout()).toContain('LLM calls: none');
|
||||
expect(io.stdout()).toContain('Your KLO project files are at:');
|
||||
expect(io.stdout()).toContain(join(tmpdir(), 'klo-demo-'));
|
||||
expect(io.stdout()).toContain('klo serve --mcp stdio');
|
||||
expect(io.stdout()).not.toContain(['klo', 'mcp'].join(' '));
|
||||
expect(io.stdout()).not.toContain('deterministic');
|
||||
});
|
||||
|
||||
it('degrades default visual seeded demo to plain output when TERM is dumb', async () => {
|
||||
const testIo = makeIo({ isTTY: true, columns: 120 });
|
||||
|
||||
await expect(
|
||||
runKloDemo(
|
||||
{ command: 'seeded', projectDir: tempDir, outputMode: 'viz', inputMode: 'disabled' },
|
||||
testIo.io,
|
||||
{ env: { ...process.env, TERM: 'dumb' } },
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(testIo.stdout()).toContain('Mode: seeded');
|
||||
expect(testIo.stdout()).toContain('LLM calls: none');
|
||||
expect(testIo.stderr()).toContain(
|
||||
'Visualization requested but TERM=dumb does not support the visual renderer; printing plain output.',
|
||||
);
|
||||
});
|
||||
|
||||
it('prints demo inspect as plain text and JSON', async () => {
|
||||
const seededIo = makeIo();
|
||||
await expect(
|
||||
runKloDemo({ command: 'seeded', projectDir: tempDir, outputMode: 'plain', inputMode: 'disabled' }, seededIo.io),
|
||||
).resolves.toBe(0);
|
||||
|
||||
const plainIo = makeIo();
|
||||
await expect(
|
||||
runKloDemo({ command: 'inspect', projectDir: tempDir, outputMode: 'plain', inputMode: 'disabled' }, plainIo.io),
|
||||
).resolves.toBe(0);
|
||||
expect(plainIo.stdout()).toContain('Mode: seeded');
|
||||
expect(plainIo.stdout()).toContain('Semantic-layer sources:');
|
||||
|
||||
const jsonIo = makeIo();
|
||||
await expect(
|
||||
runKloDemo({ command: 'inspect', projectDir: tempDir, outputMode: 'json', inputMode: 'disabled' }, jsonIo.io),
|
||||
).resolves.toBe(0);
|
||||
const parsed = JSON.parse(jsonIo.stdout());
|
||||
expect(parsed).toMatchObject({
|
||||
projectDir: tempDir,
|
||||
mode: 'seeded',
|
||||
status: { status: 'ready', missing: [] },
|
||||
sourceBundle: {
|
||||
warehouse: { tableCount: 8, totalRows: 11234 },
|
||||
dbt: { modelCount: 3, sourceTableCount: 8 },
|
||||
bi: { exploreCount: 5, dashboardCount: 2 },
|
||||
notion: { pageCount: 8 },
|
||||
},
|
||||
generatedOutputs: {
|
||||
semanticLayer: { manifestSourceCount: 6, fileCount: 6 },
|
||||
knowledge: { manifestPageCount: 10, fileCount: 10 },
|
||||
links: { manifestLinkCount: 23, linkCount: 23 },
|
||||
reports: { primaryPath: 'reports/seeded-demo-report.json', fileCount: 1 },
|
||||
},
|
||||
modeMetadata: {
|
||||
mode: 'seeded',
|
||||
source: 'packaged demo project',
|
||||
generatedContext: 'prebuilt from bundled assets',
|
||||
llmCalls: 'none',
|
||||
},
|
||||
nextCommands: KLO_NEXT_STEP_COMMANDS,
|
||||
});
|
||||
expect(parsed.generatedOutputs.replays.fileCount).toBeGreaterThanOrEqual(3);
|
||||
expect(jsonIo.stderr()).toBe('');
|
||||
});
|
||||
|
||||
it('routes top-level full mode and prints memory-flow plus final summary', async () => {
|
||||
const testIo = makeIo({ isTTY: true });
|
||||
const runFullDemo = vi.fn().mockResolvedValue(fakeFullResult(tempDir));
|
||||
await ensureDemoProject({ projectDir: tempDir, force: false });
|
||||
|
||||
await expect(
|
||||
runKloDemo({ command: 'full', projectDir: tempDir, outputMode: 'viz', inputMode: 'disabled' }, testIo.io, {
|
||||
env: {},
|
||||
runFullDemo,
|
||||
}),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(runFullDemo).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
projectDir: tempDir,
|
||||
env: {},
|
||||
onMemoryFlowChange: expect.any(Function),
|
||||
}),
|
||||
);
|
||||
expect(testIo.stdout()).toContain('KLO memory flow orbit_demo/live-database done');
|
||||
expect(testIo.stdout()).toContain('Full demo ingest: done');
|
||||
expect(testIo.stdout()).toContain('Next: klo setup demo inspect');
|
||||
expect(testIo.stdout()).toContain('Shows the files, semantic-layer sources, and memory KLO just produced.');
|
||||
});
|
||||
|
||||
it('streams live memory-flow snapshots for full demo viz and then prints final summary', async () => {
|
||||
const testIo = makeIo({ isTTY: true, columns: 120 });
|
||||
const liveSession = {
|
||||
update: vi.fn(),
|
||||
close: vi.fn(),
|
||||
isClosed: vi.fn(() => false),
|
||||
};
|
||||
const startLiveMemoryFlow = vi.fn(async (_input: MemoryFlowReplayInput, _io: unknown) => liveSession);
|
||||
const runFullDemo = vi.fn(
|
||||
async (options: { projectDir: string; onMemoryFlowChange?: (snapshot: MemoryFlowReplayInput) => void }) => {
|
||||
options.onMemoryFlowChange?.({
|
||||
...fakeFullResult(tempDir).replay,
|
||||
status: 'running',
|
||||
events: [{ type: 'source_acquired', adapter: 'live-database', trigger: 'demo_full', fileCount: 7 }],
|
||||
});
|
||||
return fakeFullResult(tempDir);
|
||||
},
|
||||
);
|
||||
await ensureDemoProject({ projectDir: tempDir, force: false });
|
||||
|
||||
await expect(
|
||||
runKloDemo({ command: 'full', projectDir: tempDir, outputMode: 'viz' }, testIo.io, {
|
||||
env: { ANTHROPIC_API_KEY: 'sk-ant-test' }, // pragma: allowlist secret
|
||||
prompts: createTestDemoPromptAdapter({ choices: ['reuse'] }),
|
||||
runFullDemo,
|
||||
startLiveMemoryFlow,
|
||||
}),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(startLiveMemoryFlow).toHaveBeenCalledTimes(1);
|
||||
expect(liveSession.update).toHaveBeenCalledTimes(1);
|
||||
expect(liveSession.close).toHaveBeenCalledTimes(1);
|
||||
expect(testIo.stdout()).not.toContain('Memory-flow summary: done');
|
||||
expect(testIo.stdout()).toContain('KLO finished ingesting your data');
|
||||
expect(testIo.stdout()).toContain('klo sl list');
|
||||
expect(testIo.stdout()).toContain('klo wiki list');
|
||||
expect(testIo.stdout()).toContain('klo serve --mcp stdio --user-id local');
|
||||
expect(testIo.stdout()).not.toContain(['klo', 'ask'].join(' '));
|
||||
expect(testIo.stdout()).not.toContain(['klo', 'mcp'].join(' '));
|
||||
});
|
||||
|
||||
it('uses plain progress for full demo viz when stdin raw mode is unavailable', async () => {
|
||||
const testIo = makeIo({ isTTY: true, rawMode: false, columns: 120 });
|
||||
const liveSession = {
|
||||
update: vi.fn(),
|
||||
close: vi.fn(),
|
||||
isClosed: vi.fn(() => false),
|
||||
};
|
||||
const startLiveMemoryFlow = vi.fn(async (_input: MemoryFlowReplayInput, _io: unknown) => liveSession);
|
||||
const runFullDemo = vi.fn(
|
||||
async (options: { projectDir: string; onMemoryFlowChange?: (snapshot: MemoryFlowReplayInput) => void }) => {
|
||||
options.onMemoryFlowChange?.({
|
||||
...fakeFullResult(tempDir).replay,
|
||||
status: 'running',
|
||||
events: [{ type: 'source_acquired', adapter: 'live-database', trigger: 'demo_full', fileCount: 7 }],
|
||||
});
|
||||
return fakeFullResult(tempDir);
|
||||
},
|
||||
);
|
||||
await ensureDemoProject({ projectDir: tempDir, force: false });
|
||||
|
||||
await expect(
|
||||
runKloDemo({ command: 'full', projectDir: tempDir, outputMode: 'viz' }, testIo.io, {
|
||||
env: { ANTHROPIC_API_KEY: 'sk-ant-test' }, // pragma: allowlist secret
|
||||
prompts: createTestDemoPromptAdapter({ choices: ['reuse'] }),
|
||||
runFullDemo,
|
||||
startLiveMemoryFlow,
|
||||
}),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(startLiveMemoryFlow).not.toHaveBeenCalled();
|
||||
expect(runFullDemo).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
onMemoryFlowChange: expect.any(Function),
|
||||
}),
|
||||
);
|
||||
expect(testIo.stdout()).toContain('[connect] Connected live-database - 7 database files (demo_full)');
|
||||
expect(testIo.stdout()).toContain('Full demo ingest: done');
|
||||
expect(testIo.stdout()).not.toContain('KLO memory flow');
|
||||
expect(testIo.stderr()).toContain(
|
||||
'Visualization requested but stdin raw mode is unavailable; printing plain output.',
|
||||
);
|
||||
});
|
||||
|
||||
it('streams plain-text progress lines for full demo when no live TUI is active', async () => {
|
||||
const testIo = makeIo();
|
||||
const runFullDemo = vi.fn(
|
||||
async (options: { projectDir: string; onMemoryFlowChange?: (snapshot: MemoryFlowReplayInput) => void }) => {
|
||||
const baseSnapshot = fakeFullResult(tempDir).replay;
|
||||
options.onMemoryFlowChange?.({
|
||||
...baseSnapshot,
|
||||
status: 'running',
|
||||
events: [{ type: 'source_acquired', adapter: 'live-database', trigger: 'manual_resync', fileCount: 7 }],
|
||||
});
|
||||
options.onMemoryFlowChange?.({
|
||||
...baseSnapshot,
|
||||
status: 'running',
|
||||
events: [
|
||||
{ type: 'source_acquired', adapter: 'live-database', trigger: 'manual_resync', fileCount: 7 },
|
||||
{ type: 'diff_computed', added: 0, modified: 0, deleted: 0, unchanged: 7 },
|
||||
],
|
||||
});
|
||||
return fakeFullResult(tempDir);
|
||||
},
|
||||
);
|
||||
await ensureDemoProject({ projectDir: tempDir, force: false });
|
||||
|
||||
await expect(
|
||||
runKloDemo(
|
||||
{ command: 'full', projectDir: tempDir, outputMode: 'plain', inputMode: 'disabled' },
|
||||
testIo.io,
|
||||
{ env: { ANTHROPIC_API_KEY: 'sk-ant-test' }, runFullDemo }, // pragma: allowlist secret
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
const stdout = testIo.stdout();
|
||||
expect(stdout).toContain('[connect] Connected live-database - 7 database files (manual_resync)');
|
||||
expect(stdout).toContain('[diff] Tables: =7 unchanged');
|
||||
expect(stdout).toContain('Full demo ingest: done');
|
||||
});
|
||||
|
||||
it('skips plain progress lines for json output mode', async () => {
|
||||
const testIo = makeIo();
|
||||
const runFullDemo = vi.fn(
|
||||
async (options: { projectDir: string; onMemoryFlowChange?: (snapshot: MemoryFlowReplayInput) => void }) => {
|
||||
expect(options.onMemoryFlowChange).toBeUndefined();
|
||||
return fakeFullResult(tempDir);
|
||||
},
|
||||
);
|
||||
await ensureDemoProject({ projectDir: tempDir, force: false });
|
||||
|
||||
await expect(
|
||||
runKloDemo(
|
||||
{ command: 'full', projectDir: tempDir, outputMode: 'json', inputMode: 'disabled' },
|
||||
testIo.io,
|
||||
{ env: { ANTHROPIC_API_KEY: 'sk-ant-test' }, runFullDemo }, // pragma: allowlist secret
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
expect(testIo.stdout()).not.toContain('[connect]');
|
||||
expect(testIo.stdout()).not.toContain('[snapshot]');
|
||||
});
|
||||
|
||||
it('routes demo ingest full mode', async () => {
|
||||
const testIo = makeIo();
|
||||
const runFullDemo = vi.fn().mockResolvedValue(fakeFullResult(tempDir));
|
||||
await ensureDemoProject({ projectDir: tempDir, force: false });
|
||||
|
||||
await expect(
|
||||
runKloDemo(
|
||||
{ command: 'ingest', mode: 'full', projectDir: tempDir, outputMode: 'plain', inputMode: 'disabled' },
|
||||
testIo.io,
|
||||
{ env: {}, runFullDemo },
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(testIo.stdout()).toContain('Full demo ingest: done');
|
||||
});
|
||||
|
||||
it('saves full-demo replay output for the next demo replay command', async () => {
|
||||
const tempDir = await mkdtemp(join(tmpdir(), 'klo-demo-full-replay-'));
|
||||
await ensureDemoProject({ projectDir: tempDir, force: false });
|
||||
const io = makeIo();
|
||||
|
||||
await expect(
|
||||
runKloDemo(
|
||||
{ command: 'full', projectDir: tempDir, outputMode: 'plain', inputMode: 'disabled' },
|
||||
io.io,
|
||||
{
|
||||
env: { ANTHROPIC_API_KEY: 'sk-ant-test' }, // pragma: allowlist secret
|
||||
runFullDemo: vi.fn(async () => fakeFullResult(tempDir)),
|
||||
},
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
const replayIo = makeIo();
|
||||
await expect(
|
||||
runKloDemo({ command: 'replay', projectDir: tempDir, outputMode: 'json', inputMode: 'disabled' }, replayIo.io),
|
||||
).resolves.toBe(0);
|
||||
expect(JSON.parse(replayIo.stdout())).toMatchObject({
|
||||
runId: 'run-full',
|
||||
metadata: { mode: 'full', origin: 'captured' },
|
||||
});
|
||||
});
|
||||
|
||||
it('routes demo ingest seeded mode through the seeded path', async () => {
|
||||
const testIo = makeIo();
|
||||
|
||||
await expect(
|
||||
runKloDemo(
|
||||
{ command: 'ingest', mode: 'seeded', projectDir: tempDir, outputMode: 'plain', inputMode: 'disabled' },
|
||||
testIo.io,
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(testIo.stdout()).toContain('Mode: seeded');
|
||||
expect(testIo.stdout()).toContain('LLM calls: none');
|
||||
});
|
||||
|
||||
it('routes demo doctor through the doctor module', async () => {
|
||||
const testIo = makeIo();
|
||||
const runDoctor = vi.fn().mockResolvedValue(0);
|
||||
|
||||
await expect(
|
||||
runKloDemo(
|
||||
{
|
||||
command: 'doctor',
|
||||
projectDir: tempDir,
|
||||
outputMode: 'plain',
|
||||
inputMode: 'disabled',
|
||||
},
|
||||
testIo.io,
|
||||
{ runDoctor },
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(runDoctor).toHaveBeenCalledWith(
|
||||
{
|
||||
command: 'demo',
|
||||
projectDir: tempDir,
|
||||
outputMode: 'plain',
|
||||
inputMode: 'disabled',
|
||||
},
|
||||
testIo.io,
|
||||
);
|
||||
});
|
||||
|
||||
it('resets the demo project only when force is explicit', async () => {
|
||||
await ensureDemoProject({ projectDir: tempDir, force: false });
|
||||
await rm(join(tempDir, 'demo.db'), { force: true });
|
||||
|
||||
const rejected = makeIo();
|
||||
await expect(
|
||||
runKloDemo({ command: 'reset', projectDir: tempDir, force: false, inputMode: 'disabled' }, rejected.io),
|
||||
).resolves.toBe(1);
|
||||
expect(rejected.stderr()).toContain(`klo setup demo reset is destructive; pass --force to recreate ${tempDir}`);
|
||||
|
||||
const accepted = makeIo();
|
||||
await expect(
|
||||
runKloDemo({ command: 'reset', projectDir: tempDir, force: true, inputMode: 'disabled' }, accepted.io),
|
||||
).resolves.toBe(0);
|
||||
expect(accepted.stdout()).toContain(`Demo project reset: ${tempDir}`);
|
||||
});
|
||||
|
||||
it('rehydrates seeded assets after reset --force', async () => {
|
||||
const resetIo = makeIo();
|
||||
await expect(
|
||||
runKloDemo({ command: 'reset', projectDir: tempDir, force: true, inputMode: 'disabled' }, resetIo.io),
|
||||
).resolves.toBe(0);
|
||||
|
||||
const seededIo = makeIo();
|
||||
await expect(
|
||||
runKloDemo(
|
||||
{ command: 'seeded', projectDir: tempDir, outputMode: 'plain', inputMode: 'disabled' },
|
||||
seededIo.io,
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(seededIo.stdout()).toContain('Status: ready');
|
||||
expect(seededIo.stdout()).toContain('Semantic-layer sources: 6 manifest, 6 files');
|
||||
expect(seededIo.stdout()).toContain('Knowledge pages: 10 manifest, 10 files');
|
||||
expect(seededIo.stdout()).not.toContain('Status: corrupt');
|
||||
expect(seededIo.stdout()).not.toContain('Semantic-layer sources: 6 manifest, 0 files');
|
||||
});
|
||||
|
||||
it('fails corrupted demo projects in no-input mode with reset guidance', async () => {
|
||||
await ensureDemoProject({ projectDir: tempDir, force: false });
|
||||
await rm(join(tempDir, 'demo.db'), { force: true });
|
||||
const testIo = makeIo();
|
||||
|
||||
await expect(
|
||||
runKloDemo({ command: 'replay', projectDir: tempDir, outputMode: 'plain', inputMode: 'disabled' }, testIo.io),
|
||||
).resolves.toBe(1);
|
||||
|
||||
expect(testIo.stderr()).toContain(`Demo project is not ready at ${tempDir}: missing demo.db`);
|
||||
expect(testIo.stderr()).toContain(`klo setup demo reset --project-dir ${tempDir} --force --no-input`);
|
||||
});
|
||||
|
||||
it('uses a process-local Anthropic key from the interactive prompt', async () => {
|
||||
const testIo = makeIo({ isTTY: true, columns: 120 });
|
||||
const runFullDemo = vi.fn().mockResolvedValue(fakeFullResult(tempDir));
|
||||
await ensureDemoProject({ projectDir: tempDir, force: false });
|
||||
|
||||
await expect(
|
||||
runKloDemo(
|
||||
{ command: 'full', projectDir: tempDir, outputMode: 'plain' },
|
||||
testIo.io,
|
||||
{
|
||||
env: {},
|
||||
prompts: createTestDemoPromptAdapter({
|
||||
choices: ['reuse', 'process_key'],
|
||||
passwords: ['sk-ant-process'], // pragma: allowlist secret
|
||||
}),
|
||||
runFullDemo,
|
||||
},
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(runFullDemo).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
projectDir: tempDir,
|
||||
env: { ANTHROPIC_API_KEY: 'sk-ant-process' }, // pragma: allowlist secret
|
||||
onMemoryFlowChange: expect.any(Function),
|
||||
}),
|
||||
);
|
||||
expect(await readFile(join(tempDir, 'klo.yaml'), 'utf-8')).toContain('api_key: env:ANTHROPIC_API_KEY');
|
||||
});
|
||||
|
||||
it('routes an interactive missing-key choice to seeded mode', async () => {
|
||||
const testIo = makeIo({ isTTY: true, columns: 120 });
|
||||
const runFullDemo = vi.fn();
|
||||
await ensureDemoProject({ projectDir: tempDir, force: false });
|
||||
|
||||
await expect(
|
||||
runKloDemo(
|
||||
{ command: 'full', projectDir: tempDir, outputMode: 'plain' },
|
||||
testIo.io,
|
||||
{
|
||||
env: {},
|
||||
prompts: createTestDemoPromptAdapter({ choices: ['reuse', 'seeded'] }),
|
||||
runFullDemo,
|
||||
},
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(runFullDemo).not.toHaveBeenCalled();
|
||||
expect(testIo.stdout()).toContain('Mode: seeded');
|
||||
expect(testIo.stdout()).toContain('LLM calls: none');
|
||||
expect(testIo.stdout()).not.toContain('deterministic');
|
||||
});
|
||||
|
||||
it('routes missing full-mode credentials to seeded when the interactive user chooses the no-LLM demo', async () => {
|
||||
const testIo = makeIo({ isTTY: true });
|
||||
|
||||
await expect(
|
||||
runKloDemo(
|
||||
{ command: 'full', projectDir: tempDir, outputMode: 'plain' },
|
||||
testIo.io,
|
||||
{
|
||||
env: { ...process.env, ANTHROPIC_API_KEY: '' },
|
||||
prompts: createTestDemoPromptAdapter({ choices: ['seeded'] }),
|
||||
},
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(testIo.stdout()).toContain('Mode: seeded');
|
||||
expect(testIo.stdout()).toContain('LLM calls: none');
|
||||
expect(testIo.stdout()).not.toContain('deterministic');
|
||||
});
|
||||
|
||||
it('routes an interactive missing-key choice to replay mode', async () => {
|
||||
const testIo = makeIo({ isTTY: true, columns: 120 });
|
||||
const runFullDemo = vi.fn();
|
||||
await ensureDemoProject({ projectDir: tempDir, force: false });
|
||||
|
||||
await expect(
|
||||
runKloDemo(
|
||||
{ command: 'full', projectDir: tempDir, outputMode: 'viz' },
|
||||
testIo.io,
|
||||
{
|
||||
env: {},
|
||||
prompts: createTestDemoPromptAdapter({ choices: ['reuse', 'replay'] }),
|
||||
runFullDemo,
|
||||
},
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(runFullDemo).not.toHaveBeenCalled();
|
||||
expect(testIo.stdout()).toContain('KLO memory flow');
|
||||
expect(testIo.stdout()).toContain('done');
|
||||
});
|
||||
});
|
||||
544
packages/cli/src/demo.ts
Normal file
544
packages/cli/src/demo.ts
Normal file
|
|
@ -0,0 +1,544 @@
|
|||
import {
|
||||
buildMemoryFlowViewModel,
|
||||
formatMemoryFlowFinalSummary,
|
||||
renderMemoryFlowReplay,
|
||||
type MemoryFlowReplayInput,
|
||||
} from '@klo/context/ingest/memory-flow';
|
||||
import { resolveKloConfigReference } from '@klo/context/core';
|
||||
import { loadKloProject } from '@klo/context/project';
|
||||
import {
|
||||
DEMO_ADAPTER,
|
||||
DEMO_CONNECTION_ID,
|
||||
DEMO_FULL_JOB_ID,
|
||||
ensureDemoProject,
|
||||
loadProjectDemoReplay,
|
||||
resetDemoProject,
|
||||
} from './demo-assets.js';
|
||||
import { writeDemoReplay } from './demo-replay-store.js';
|
||||
import {
|
||||
formatDemoInspect,
|
||||
formatDemoScanSummary,
|
||||
inspectDemoProject,
|
||||
runDemoScan,
|
||||
} from './demo-scan.js';
|
||||
import {
|
||||
formatSeededInspect,
|
||||
inspectSeededProject,
|
||||
runDemoSeeded,
|
||||
} from './demo-seeded.js';
|
||||
import { buildFullDemoReplay, formatCleanDemoSummary, formatFullDemoSummary, fullDemoCredentialStatus, runDemoFull } from './demo-full.js';
|
||||
import { createPlainProgressEmitter } from './demo-progress.js';
|
||||
import {
|
||||
chooseDemoProjectForInteractiveRun,
|
||||
createClackDemoPromptAdapter,
|
||||
resolveFullCredentialDecision,
|
||||
type DemoPromptAdapter,
|
||||
} from './demo-interaction.js';
|
||||
import type { KloDoctorArgs } from './doctor.js';
|
||||
import {
|
||||
renderMemoryFlowTui,
|
||||
startLiveMemoryFlowTui,
|
||||
type KloMemoryFlowTuiIo,
|
||||
type MemoryFlowTuiLiveSession,
|
||||
} from './memory-flow-tui.js';
|
||||
import {
|
||||
rendererUnavailableVizFallback,
|
||||
resolveVizFallback,
|
||||
warnVizFallbackOnce,
|
||||
} from './viz-fallback.js';
|
||||
import { profileMark } from './startup-profile.js';
|
||||
import { formatNextStepLines } from './next-steps.js';
|
||||
|
||||
profileMark('module:demo');
|
||||
|
||||
export type KloDemoOutputMode = 'plain' | 'json' | 'viz';
|
||||
export type KloDemoInputMode = 'auto' | 'disabled';
|
||||
export type KloDemoMode = 'full' | 'seeded';
|
||||
|
||||
export type KloDemoArgs =
|
||||
| { command: 'init'; projectDir: string; force: boolean; inputMode?: KloDemoInputMode }
|
||||
| { command: 'reset'; projectDir: string; force: boolean; inputMode?: KloDemoInputMode }
|
||||
| { command: 'replay'; projectDir: string; outputMode: KloDemoOutputMode; inputMode?: KloDemoInputMode }
|
||||
| { command: 'scan'; projectDir: string; inputMode?: KloDemoInputMode }
|
||||
| { command: 'inspect'; projectDir: string; outputMode: KloDemoOutputMode; inputMode?: KloDemoInputMode }
|
||||
| { command: 'doctor'; projectDir: string; outputMode: Exclude<KloDemoOutputMode, 'viz'>; inputMode?: KloDemoInputMode }
|
||||
| { command: 'seeded'; projectDir: string; outputMode: KloDemoOutputMode; inputMode?: KloDemoInputMode }
|
||||
| { command: 'full'; projectDir: string; outputMode: KloDemoOutputMode; inputMode?: KloDemoInputMode }
|
||||
| {
|
||||
command: 'ingest';
|
||||
mode: KloDemoMode;
|
||||
projectDir: string;
|
||||
outputMode: KloDemoOutputMode;
|
||||
inputMode?: KloDemoInputMode;
|
||||
};
|
||||
|
||||
export interface KloDemoIo {
|
||||
stdin?: KloMemoryFlowTuiIo['stdin'];
|
||||
stdout: { isTTY?: boolean; columns?: number; write(chunk: string): void };
|
||||
stderr: { write(chunk: string): void };
|
||||
}
|
||||
|
||||
interface KloDemoDeps {
|
||||
runFullDemo?: typeof runDemoFull;
|
||||
runDoctor?: (args: KloDoctorArgs, io: KloDemoIo) => Promise<number>;
|
||||
renderStoredMemoryFlow?: typeof renderMemoryFlowTui;
|
||||
startLiveMemoryFlow?: typeof startLiveMemoryFlowTui;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
prompts?: DemoPromptAdapter;
|
||||
}
|
||||
|
||||
const ADAPTER_PREFIXES = ['live_database_', 'metabase_', 'looker_', 'lookml_', 'metricflow_', 'notion_', 'historic_sql_', 'dbt_descriptions_'];
|
||||
const DEMO_TUI_SPEED_MULTIPLIER = 0.125;
|
||||
|
||||
function humanizeUnitKeyPlain(unitKey: string): string {
|
||||
let key = unitKey.replace(/-/g, '_');
|
||||
for (const prefix of ADAPTER_PREFIXES) {
|
||||
if (key.startsWith(prefix)) { key = key.slice(prefix.length); break; }
|
||||
}
|
||||
return key.replace(/_/g, ' ');
|
||||
}
|
||||
|
||||
function formatReplaySummary(input: MemoryFlowReplayInput): string {
|
||||
let slCount = 0;
|
||||
let wikiCount = 0;
|
||||
let chunkCount = 0;
|
||||
const unitResults: Array<{ unitKey: string; artifacts: Array<{ icon: string; text: string; hasSummary: boolean }> }> = [];
|
||||
let currentUnit: { unitKey: string; artifacts: Array<{ icon: string; text: string; hasSummary: boolean }> } | null = null;
|
||||
let conflictCount = 0;
|
||||
|
||||
for (const e of input.events) {
|
||||
if (e.type === 'chunks_planned') {
|
||||
chunkCount = e.chunkCount;
|
||||
} else if (e.type === 'work_unit_started') {
|
||||
currentUnit = { unitKey: e.unitKey, artifacts: [] };
|
||||
} else if (e.type === 'candidate_action') {
|
||||
if (e.target === 'sl') slCount++;
|
||||
else wikiCount++;
|
||||
const detail = input.details.actions.find((a) => a.key === e.key && a.unitKey === e.unitKey);
|
||||
const icon = e.target === 'sl' ? '📊' : '📝';
|
||||
const name = e.key.split('.').pop()?.replace(/[_-]/g, ' ') ?? e.key;
|
||||
const text = detail?.summary ?? name;
|
||||
currentUnit?.artifacts.push({ icon, text, hasSummary: !!detail?.summary });
|
||||
} else if (e.type === 'work_unit_finished' && currentUnit) {
|
||||
unitResults.push(currentUnit);
|
||||
currentUnit = null;
|
||||
} else if (e.type === 'reconciliation_finished') {
|
||||
conflictCount = e.conflictCount;
|
||||
}
|
||||
}
|
||||
|
||||
const lines: string[] = ['', '★ KLO finished ingesting your data', ''];
|
||||
|
||||
if (chunkCount > 0) {
|
||||
lines.push(` ✓ Analyzed ${chunkCount} business area${chunkCount === 1 ? '' : 's'}`);
|
||||
}
|
||||
|
||||
lines.push(` ✓ Reconciled — ${conflictCount > 0 ? `${conflictCount} conflict${conflictCount === 1 ? '' : 's'} resolved` : 'no conflicts'}`);
|
||||
lines.push('');
|
||||
|
||||
if (slCount > 0 || wikiCount > 0) {
|
||||
lines.push(' KLO created:');
|
||||
if (slCount > 0) lines.push(` 📊 ${slCount} query definition${slCount === 1 ? '' : 's'} — so agents can write accurate SQL for your data`);
|
||||
if (wikiCount > 0) lines.push(` 📝 ${wikiCount} knowledge page${wikiCount === 1 ? '' : 's'} — so agents understand your business context`);
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
const described = unitResults.flatMap((u) => u.artifacts).filter((a) => a.hasSummary);
|
||||
for (const a of described) {
|
||||
lines.push(` ${a.icon} ${a.text}`);
|
||||
}
|
||||
|
||||
lines.push('');
|
||||
lines.push(' What to do next:');
|
||||
lines.push(...formatNextStepLines());
|
||||
if (input.sourceDir) {
|
||||
lines.push('');
|
||||
lines.push(` Your KLO project files are at: ${input.sourceDir}`);
|
||||
}
|
||||
lines.push('');
|
||||
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
function formatPlainReplaySummary(input: MemoryFlowReplayInput): string {
|
||||
return [formatMemoryFlowFinalSummary(input).trimEnd(), '', 'What to do next:', ...formatNextStepLines(), ''].join('\n');
|
||||
}
|
||||
|
||||
function writeReplay(input: MemoryFlowReplayInput, outputMode: KloDemoOutputMode, io: KloDemoIo): void {
|
||||
if (outputMode === 'json') {
|
||||
io.stdout.write(`${JSON.stringify(input, null, 2)}\n`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (outputMode === 'plain') {
|
||||
io.stdout.write(formatPlainReplaySummary(input));
|
||||
return;
|
||||
}
|
||||
|
||||
const view = buildMemoryFlowViewModel(input);
|
||||
io.stdout.write(renderMemoryFlowReplay(view, { terminalWidth: io.stdout.columns ?? process.stdout.columns }));
|
||||
}
|
||||
|
||||
async function writeStoredReplay(
|
||||
input: MemoryFlowReplayInput,
|
||||
outputMode: KloDemoOutputMode,
|
||||
inputMode: KloDemoArgs['inputMode'],
|
||||
io: KloDemoIo,
|
||||
deps: KloDemoDeps,
|
||||
env: NodeJS.ProcessEnv,
|
||||
): Promise<void> {
|
||||
const resolvedOutputMode = effectiveDemoOutputMode(outputMode, io, env, {
|
||||
requireInput: inputMode !== 'disabled',
|
||||
});
|
||||
if (resolvedOutputMode !== 'viz') {
|
||||
writeReplay(input, resolvedOutputMode, io);
|
||||
return;
|
||||
}
|
||||
|
||||
if (inputMode !== 'disabled') {
|
||||
const renderStoredMemoryFlow = deps.renderStoredMemoryFlow ?? renderMemoryFlowTui;
|
||||
if (
|
||||
isTuiCapableDemoIo(io) &&
|
||||
(await renderStoredMemoryFlow(input, io, { speedMultiplier: DEMO_TUI_SPEED_MULTIPLIER }))
|
||||
) {
|
||||
io.stdout.write(formatReplaySummary(input));
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
writeReplay(input, resolvedOutputMode, io);
|
||||
}
|
||||
|
||||
function writeInspect(
|
||||
summary: Awaited<ReturnType<typeof inspectDemoProject>>,
|
||||
outputMode: KloDemoOutputMode,
|
||||
io: KloDemoIo,
|
||||
): void {
|
||||
if (outputMode === 'json') {
|
||||
io.stdout.write(`${JSON.stringify(summary, null, 2)}\n`);
|
||||
return;
|
||||
}
|
||||
|
||||
io.stdout.write(formatDemoInspect(summary));
|
||||
}
|
||||
|
||||
function writeFullDemo(
|
||||
result: Awaited<ReturnType<typeof runDemoFull>>,
|
||||
outputMode: KloDemoOutputMode,
|
||||
io: KloDemoIo,
|
||||
options: { liveWasRendered?: boolean; projectDir?: string } = {},
|
||||
): void {
|
||||
if (outputMode === 'json') {
|
||||
io.stdout.write(`${JSON.stringify({ report: result.report, replay: result.replay }, null, 2)}\n`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (outputMode === 'viz' && options.liveWasRendered !== true) {
|
||||
writeReplay(buildFullDemoReplay(result.report), outputMode, io);
|
||||
io.stdout.write('\n');
|
||||
}
|
||||
|
||||
if (outputMode === 'viz' && options.liveWasRendered) {
|
||||
io.stdout.write(formatCleanDemoSummary(result.report, options.projectDir ?? ''));
|
||||
return;
|
||||
}
|
||||
|
||||
if (outputMode === 'viz') {
|
||||
io.stdout.write(formatMemoryFlowFinalSummary(buildFullDemoReplay(result.report)));
|
||||
}
|
||||
|
||||
io.stdout.write(formatFullDemoSummary(result.report));
|
||||
}
|
||||
|
||||
function replayWithFullMetadata(result: Awaited<ReturnType<typeof runDemoFull>>): MemoryFlowReplayInput {
|
||||
if (result.replay.metadata) {
|
||||
return result.replay;
|
||||
}
|
||||
|
||||
return {
|
||||
...result.replay,
|
||||
metadata: {
|
||||
schemaVersion: 1,
|
||||
mode: 'full',
|
||||
origin: 'captured',
|
||||
timing: 'captured',
|
||||
capturedAt: result.report.createdAt,
|
||||
sourceReportId: result.report.id,
|
||||
sourceReportPath: result.report.id,
|
||||
fallbackReason: null,
|
||||
},
|
||||
reportId: result.replay.reportId ?? result.report.id,
|
||||
reportPath: result.replay.reportPath ?? result.report.id,
|
||||
};
|
||||
}
|
||||
|
||||
function pickMemoryFlowProgress(
|
||||
liveSession: MemoryFlowTuiLiveSession | null,
|
||||
outputMode: KloDemoOutputMode,
|
||||
io: KloDemoIo,
|
||||
): ((snapshot: MemoryFlowReplayInput) => void) | undefined {
|
||||
if (liveSession) {
|
||||
return (snapshot: MemoryFlowReplayInput) => {
|
||||
if (!liveSession.isClosed()) {
|
||||
liveSession.update(snapshot);
|
||||
}
|
||||
};
|
||||
}
|
||||
if (outputMode === 'json') {
|
||||
return undefined;
|
||||
}
|
||||
return createPlainProgressEmitter(io);
|
||||
}
|
||||
|
||||
function isTuiCapableDemoIo(io: KloDemoIo): io is KloDemoIo & KloMemoryFlowTuiIo {
|
||||
return (
|
||||
io.stdin?.isTTY === true &&
|
||||
io.stdout.isTTY === true &&
|
||||
typeof io.stdin.setRawMode === 'function' &&
|
||||
typeof io.stdout.write === 'function'
|
||||
);
|
||||
}
|
||||
|
||||
interface EffectiveDemoOutputModeOptions {
|
||||
requireInput?: boolean;
|
||||
}
|
||||
|
||||
function effectiveDemoOutputMode(
|
||||
outputMode: KloDemoOutputMode,
|
||||
io: KloDemoIo,
|
||||
env: NodeJS.ProcessEnv,
|
||||
options: EffectiveDemoOutputModeOptions = {},
|
||||
): KloDemoOutputMode {
|
||||
if (outputMode !== 'viz') {
|
||||
return outputMode;
|
||||
}
|
||||
|
||||
const fallback = resolveVizFallback(io, env, { requireInput: options.requireInput ?? false });
|
||||
if (!fallback.shouldDegrade) {
|
||||
return outputMode;
|
||||
}
|
||||
|
||||
warnVizFallbackOnce(io, fallback);
|
||||
return 'plain';
|
||||
}
|
||||
|
||||
function initialFullDemoMemoryFlowInput(projectDir: string): MemoryFlowReplayInput {
|
||||
return {
|
||||
runId: DEMO_FULL_JOB_ID,
|
||||
connectionId: DEMO_CONNECTION_ID,
|
||||
adapter: DEMO_ADAPTER,
|
||||
status: 'running',
|
||||
sourceDir: `${projectDir}/raw-sources/${DEMO_CONNECTION_ID}/${DEMO_ADAPTER}`,
|
||||
syncId: 'pending',
|
||||
errors: [],
|
||||
events: [],
|
||||
plannedWorkUnits: [],
|
||||
details: { actions: [], provenance: [], transcripts: [] },
|
||||
};
|
||||
}
|
||||
|
||||
async function ensureDemoProjectForCommand(projectDir: string): Promise<void> {
|
||||
await ensureDemoProject({ projectDir, force: false }).catch((error) => {
|
||||
if (error instanceof Error && error.message.includes('Demo project already exists')) {
|
||||
return null;
|
||||
}
|
||||
throw error;
|
||||
});
|
||||
}
|
||||
|
||||
async function prepareProjectForDemoCommand(args: KloDemoArgs, io: KloDemoIo, deps: KloDemoDeps): Promise<string | null> {
|
||||
if (args.command === 'init' || args.command === 'reset' || args.command === 'doctor') {
|
||||
return args.projectDir;
|
||||
}
|
||||
|
||||
const prompts = deps.prompts ?? createClackDemoPromptAdapter();
|
||||
const decision = await chooseDemoProjectForInteractiveRun({
|
||||
projectDir: args.projectDir,
|
||||
inputMode: args.inputMode,
|
||||
io,
|
||||
prompts,
|
||||
});
|
||||
|
||||
if (decision.action === 'cancel') {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (decision.reset) {
|
||||
await resetDemoProject({ projectDir: decision.projectDir, force: true });
|
||||
}
|
||||
|
||||
return decision.projectDir;
|
||||
}
|
||||
|
||||
async function runReplayDemo(
|
||||
projectDir: string,
|
||||
outputMode: KloDemoOutputMode,
|
||||
inputMode: KloDemoArgs['inputMode'],
|
||||
io: KloDemoIo,
|
||||
deps: KloDemoDeps,
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
): Promise<number> {
|
||||
await ensureDemoProjectForCommand(projectDir);
|
||||
await writeStoredReplay(await loadProjectDemoReplay(projectDir), outputMode, inputMode, io, deps, env);
|
||||
return 0;
|
||||
}
|
||||
|
||||
async function runSeededDemo(
|
||||
projectDir: string,
|
||||
outputMode: KloDemoOutputMode,
|
||||
inputMode: KloDemoArgs['inputMode'],
|
||||
io: KloDemoIo,
|
||||
deps: KloDemoDeps,
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
): Promise<number> {
|
||||
const result = await runDemoSeeded({ projectDir });
|
||||
const resolvedOutputMode = effectiveDemoOutputMode(outputMode, io, env, {
|
||||
requireInput: inputMode !== 'disabled',
|
||||
});
|
||||
|
||||
if (resolvedOutputMode === 'json') {
|
||||
io.stdout.write(`${JSON.stringify({ replay: result.replay, inspect: result.inspect }, null, 2)}\n`);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (resolvedOutputMode === 'viz') {
|
||||
await writeStoredReplay(result.replay, resolvedOutputMode, inputMode, io, deps, env);
|
||||
} else {
|
||||
writeReplay(result.replay, resolvedOutputMode, io);
|
||||
io.stdout.write('\n');
|
||||
io.stdout.write(formatSeededInspect(result.inspect));
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
export async function runKloDemo(args: KloDemoArgs, io: KloDemoIo = process, deps: KloDemoDeps = {}): Promise<number> {
|
||||
try {
|
||||
if (args.command === 'init') {
|
||||
const result = await ensureDemoProject({ projectDir: args.projectDir, force: args.force });
|
||||
io.stdout.write(`Demo project: ${result.projectDir}\n`);
|
||||
io.stdout.write(`Config: ${result.configPath}\n`);
|
||||
io.stdout.write(`Database: ${result.databasePath}\n`);
|
||||
io.stdout.write(`Replay: ${result.replayPath}\n`);
|
||||
io.stdout.write('Next: klo setup demo --no-input\n');
|
||||
io.stdout.write(' Runs the pre-seeded demo without calling the LLM.\n');
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (args.command === 'reset') {
|
||||
const result = await resetDemoProject({ projectDir: args.projectDir, force: args.force });
|
||||
io.stdout.write(`Demo project reset: ${result.projectDir}\n`);
|
||||
io.stdout.write(`Config: ${result.configPath}\n`);
|
||||
io.stdout.write(`Database: ${result.databasePath}\n`);
|
||||
io.stdout.write(`Replay: ${result.replayPath}\n`);
|
||||
io.stdout.write('Next: klo setup demo --mode full\n');
|
||||
io.stdout.write(' Runs the full AI-backed pass with your LLM provider.\n');
|
||||
return 0;
|
||||
}
|
||||
|
||||
const preparedProjectDir = await prepareProjectForDemoCommand(args, io, deps);
|
||||
if (preparedProjectDir === null) {
|
||||
return 1;
|
||||
}
|
||||
const env = deps.env ?? process.env;
|
||||
|
||||
if (args.command === 'scan') {
|
||||
const { result } = await runDemoScan({ projectDir: preparedProjectDir });
|
||||
io.stdout.write(formatDemoScanSummary(result.report));
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (args.command === 'seeded' || (args.command === 'ingest' && args.mode === 'seeded')) {
|
||||
return await runSeededDemo(preparedProjectDir, args.outputMode, args.inputMode, io, deps, env);
|
||||
}
|
||||
|
||||
if (args.command === 'full' || (args.command === 'ingest' && args.mode === 'full')) {
|
||||
const executeFullDemo = deps.runFullDemo ?? runDemoFull;
|
||||
await ensureDemoProjectForCommand(preparedProjectDir);
|
||||
const project = await loadKloProject({ projectDir: preparedProjectDir });
|
||||
const credentialStatus = fullDemoCredentialStatus(project, env);
|
||||
const credentialDecision = await resolveFullCredentialDecision({
|
||||
needsAnthropicKey:
|
||||
credentialStatus.status === 'missing-anthropic-key' &&
|
||||
project.config.llm.provider.backend === 'anthropic' &&
|
||||
!resolveKloConfigReference(project.config.llm.provider.anthropic?.api_key, env),
|
||||
inputMode: args.inputMode,
|
||||
io,
|
||||
env,
|
||||
prompts: deps.prompts ?? createClackDemoPromptAdapter(),
|
||||
});
|
||||
|
||||
if (credentialDecision.action === 'cancel') {
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (credentialDecision.action === 'run-mode') {
|
||||
return credentialDecision.mode === 'seeded'
|
||||
? await runSeededDemo(preparedProjectDir, args.outputMode, args.inputMode, io, deps, env)
|
||||
: await runReplayDemo(preparedProjectDir, args.outputMode, args.inputMode, io, deps, env);
|
||||
}
|
||||
|
||||
let liveSession: MemoryFlowTuiLiveSession | null = null;
|
||||
let liveWasRendered = false;
|
||||
const startLiveMemoryFlow = deps.startLiveMemoryFlow ?? startLiveMemoryFlowTui;
|
||||
let fullOutputMode = effectiveDemoOutputMode(args.outputMode, io, env, {
|
||||
requireInput: args.inputMode !== 'disabled',
|
||||
});
|
||||
const shouldUseLiveViz = fullOutputMode === 'viz' && args.inputMode !== 'disabled';
|
||||
|
||||
if (shouldUseLiveViz && isTuiCapableDemoIo(io)) {
|
||||
liveSession = await startLiveMemoryFlow(initialFullDemoMemoryFlowInput(preparedProjectDir), io);
|
||||
liveWasRendered = liveSession !== null;
|
||||
} else if (shouldUseLiveViz) {
|
||||
warnVizFallbackOnce(io, rendererUnavailableVizFallback());
|
||||
fullOutputMode = 'plain';
|
||||
}
|
||||
|
||||
const onMemoryFlowChange = pickMemoryFlowProgress(liveSession, fullOutputMode, io);
|
||||
const result = await executeFullDemo({
|
||||
projectDir: preparedProjectDir,
|
||||
env: credentialDecision.env,
|
||||
...(onMemoryFlowChange ? { onMemoryFlowChange } : {}),
|
||||
});
|
||||
await writeDemoReplay(preparedProjectDir, replayWithFullMetadata(result), { label: 'full' });
|
||||
liveSession?.close();
|
||||
writeFullDemo(result, fullOutputMode, io, { liveWasRendered, projectDir: preparedProjectDir });
|
||||
if (fullOutputMode !== 'json' && !liveWasRendered) {
|
||||
io.stdout.write(formatDemoInspect(await inspectDemoProject(preparedProjectDir)));
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (args.command === 'inspect') {
|
||||
const seededInspect = await inspectSeededProject(preparedProjectDir).catch(() => null);
|
||||
if (seededInspect?.mode === 'seeded') {
|
||||
if (args.outputMode === 'json') {
|
||||
io.stdout.write(`${JSON.stringify(seededInspect, null, 2)}\n`);
|
||||
} else {
|
||||
io.stdout.write(formatSeededInspect(seededInspect));
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
writeInspect(await inspectDemoProject(preparedProjectDir), args.outputMode, io);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (args.command === 'doctor') {
|
||||
const { runKloDoctor } = await import('./doctor.js');
|
||||
const executeDoctor = deps.runDoctor ?? runKloDoctor;
|
||||
return await executeDoctor(
|
||||
{
|
||||
command: 'demo',
|
||||
projectDir: args.projectDir,
|
||||
outputMode: args.outputMode,
|
||||
...(args.inputMode ? { inputMode: args.inputMode } : {}),
|
||||
},
|
||||
io,
|
||||
);
|
||||
}
|
||||
|
||||
return await runReplayDemo(preparedProjectDir, args.outputMode, args.inputMode, io, deps, env);
|
||||
} catch (error) {
|
||||
io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
670
packages/cli/src/dev.test.ts
Normal file
670
packages/cli/src/dev.test.ts
Normal file
|
|
@ -0,0 +1,670 @@
|
|||
import { describe, expect, it, vi } from 'vitest';
|
||||
import { runKloCli } from './index.js';
|
||||
|
||||
function makeIo() {
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
return {
|
||||
io: {
|
||||
stdout: {
|
||||
write: (chunk: string) => {
|
||||
stdout += chunk;
|
||||
},
|
||||
},
|
||||
stderr: {
|
||||
write: (chunk: string) => {
|
||||
stderr += chunk;
|
||||
},
|
||||
},
|
||||
},
|
||||
stdout: () => stdout,
|
||||
stderr: () => stderr,
|
||||
};
|
||||
}
|
||||
|
||||
describe('dev Commander tree', () => {
|
||||
it('prints visible dev help with only supported low-level command groups', async () => {
|
||||
const testIo = makeIo();
|
||||
|
||||
await expect(runKloCli(['dev', '--help'], testIo.io)).resolves.toBe(0);
|
||||
|
||||
expect(testIo.stdout()).toContain('Usage: klo dev [options] [command]');
|
||||
for (const command of ['init', 'doctor', 'scan', 'ingest', 'mapping']) {
|
||||
expect(testIo.stdout()).toContain(command);
|
||||
}
|
||||
for (const removed of [
|
||||
'knowledge',
|
||||
'model',
|
||||
'replay',
|
||||
'report',
|
||||
'status',
|
||||
'artifacts',
|
||||
'config',
|
||||
'tools',
|
||||
'daemon',
|
||||
]) {
|
||||
expect(testIo.stdout()).not.toContain(`${removed} `);
|
||||
}
|
||||
expect(testIo.stderr()).toBe('');
|
||||
});
|
||||
|
||||
it('keeps dev callable while hiding it from root command rows', async () => {
|
||||
const testIo = makeIo();
|
||||
|
||||
await expect(runKloCli(['--help'], testIo.io)).resolves.toBe(0);
|
||||
|
||||
expect(testIo.stdout()).toContain('Advanced:');
|
||||
expect(testIo.stdout()).toContain('klo dev');
|
||||
expect(testIo.stdout()).not.toContain('dev Low-level diagnostics');
|
||||
expect(testIo.stderr()).toBe('');
|
||||
});
|
||||
|
||||
it('keeps project scaffolding under dev init', async () => {
|
||||
const { mkdtemp, readFile, rm } = await import('node:fs/promises');
|
||||
const { tmpdir } = await import('node:os');
|
||||
const { join } = await import('node:path');
|
||||
const tempDir = await mkdtemp(join(tmpdir(), 'klo-dev-init-'));
|
||||
const projectDir = join(tempDir, 'warehouse');
|
||||
const testIo = makeIo();
|
||||
|
||||
try {
|
||||
await expect(runKloCli(['dev', 'init', projectDir, '--name', 'warehouse'], testIo.io)).resolves.toBe(0);
|
||||
|
||||
expect(testIo.stdout()).toContain(`Initialized KLO project at ${projectDir}`);
|
||||
await expect(readFile(join(projectDir, 'klo.yaml'), 'utf-8')).resolves.toContain('project: warehouse');
|
||||
expect(testIo.stderr()).toBe('');
|
||||
} finally {
|
||||
await rm(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('uses global project-dir for dev init when the positional directory is omitted', async () => {
|
||||
const { mkdtemp, rm } = await import('node:fs/promises');
|
||||
const { tmpdir } = await import('node:os');
|
||||
const { join } = await import('node:path');
|
||||
const tempDir = await mkdtemp(join(tmpdir(), 'klo-dev-init-global-'));
|
||||
const projectDir = join(tempDir, 'global-init');
|
||||
const testIo = makeIo();
|
||||
|
||||
try {
|
||||
await expect(
|
||||
runKloCli(['--project-dir', projectDir, 'dev', 'init', '--name', 'global-init'], testIo.io),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(testIo.stdout()).toContain(`Initialized KLO project at ${projectDir}`);
|
||||
expect(testIo.stderr()).toBe('');
|
||||
} finally {
|
||||
await rm(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('rejects removed dev command groups', async () => {
|
||||
for (const argv of [
|
||||
['dev', 'knowledge', 'list'],
|
||||
['dev', 'model', 'list'],
|
||||
['dev', 'artifacts'],
|
||||
]) {
|
||||
const testIo = makeIo();
|
||||
|
||||
await expect(runKloCli(argv, testIo.io)).resolves.toBe(1);
|
||||
|
||||
expect(testIo.stderr()).toMatch(/unknown command|error:/);
|
||||
}
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
argv: ['dev', 'doctor', '--help'],
|
||||
expected: ['Usage: klo dev doctor', '--json', '--no-input'],
|
||||
},
|
||||
{
|
||||
argv: ['dev', 'scan', '--help'],
|
||||
expected: [
|
||||
'Usage: klo dev scan',
|
||||
'--mode <mode>',
|
||||
'structural',
|
||||
'relationships',
|
||||
'--dry-run',
|
||||
'status',
|
||||
'report',
|
||||
'relationships',
|
||||
'relationship-apply',
|
||||
'relationship-feedback',
|
||||
'relationship-calibration',
|
||||
'relationship-thresholds',
|
||||
],
|
||||
},
|
||||
{
|
||||
argv: ['dev', 'scan', 'report', '--help'],
|
||||
expected: ['Usage: klo dev scan report [options] <runId>', '<runId>', '--json'],
|
||||
},
|
||||
{
|
||||
argv: ['dev', 'scan', 'relationships', '--help'],
|
||||
expected: [
|
||||
'Usage: klo dev scan relationships [options] <runId>',
|
||||
'--status <status>',
|
||||
'--limit <count>',
|
||||
'--accept <candidateId>',
|
||||
'--reject <candidateId>',
|
||||
'--note <text>',
|
||||
'--reviewer <name>',
|
||||
'--json',
|
||||
],
|
||||
},
|
||||
{
|
||||
argv: ['dev', 'scan', 'relationship-apply', '--help'],
|
||||
expected: [
|
||||
'Usage: klo dev scan relationship-apply [options] <runId>',
|
||||
'--all-accepted',
|
||||
'--candidate <candidateId>',
|
||||
'--dry-run',
|
||||
],
|
||||
},
|
||||
{
|
||||
argv: ['dev', 'scan', 'relationship-thresholds', '--help'],
|
||||
expected: [
|
||||
'Usage: klo dev scan relationship-thresholds [options]',
|
||||
'--connection <connectionId>',
|
||||
'--min-total-labels <count>',
|
||||
'--min-accepted-labels <count>',
|
||||
'--min-rejected-labels <count>',
|
||||
'--json',
|
||||
],
|
||||
},
|
||||
{
|
||||
argv: ['dev', 'scan', 'relationship-feedback', '--help'],
|
||||
expected: [
|
||||
'Usage: klo dev scan relationship-feedback [options]',
|
||||
'--connection <connectionId>',
|
||||
'--decision <decision>',
|
||||
'--json',
|
||||
'--jsonl',
|
||||
],
|
||||
},
|
||||
{
|
||||
argv: ['dev', 'scan', 'relationship-calibration', '--help'],
|
||||
expected: [
|
||||
'Usage: klo dev scan relationship-calibration [options]',
|
||||
'--connection <connectionId>',
|
||||
'--decision <decision>',
|
||||
'--accept-threshold <value>',
|
||||
'--review-threshold <value>',
|
||||
'--json',
|
||||
],
|
||||
},
|
||||
{
|
||||
argv: ['dev', 'ingest', 'run', '--help'],
|
||||
expected: ['Usage: klo dev ingest run [options]', '--connection-id <connectionId>', '--adapter <adapter>'],
|
||||
},
|
||||
{
|
||||
argv: ['dev', 'mapping', 'sync-state', 'set', '--help'],
|
||||
expected: ['Usage: klo dev mapping sync-state set [options] <connectionId>', '--mode <mode>'],
|
||||
},
|
||||
])('prints generated nested help for $argv', async ({ argv, expected }) => {
|
||||
const io = makeIo();
|
||||
const doctor = vi.fn(async () => 0);
|
||||
const ingest = vi.fn(async () => 0);
|
||||
const scan = vi.fn(async () => 0);
|
||||
|
||||
await expect(runKloCli(argv, io.io, { doctor, ingest, scan })).resolves.toBe(0);
|
||||
|
||||
for (const text of expected) {
|
||||
expect(io.stdout()).toContain(text);
|
||||
}
|
||||
expect(io.stderr()).toBe('');
|
||||
expect(doctor).not.toHaveBeenCalled();
|
||||
expect(ingest).not.toHaveBeenCalled();
|
||||
expect(scan).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('dispatches dev scan through Commander with injected dependencies', async () => {
|
||||
const scanIo = makeIo();
|
||||
const scan = vi.fn(async () => 0);
|
||||
|
||||
await expect(
|
||||
runKloCli(['dev', 'scan', 'warehouse', '--project-dir', '/tmp/project', '--dry-run'], scanIo.io, { scan }),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(scan).toHaveBeenCalledWith(
|
||||
{
|
||||
command: 'run',
|
||||
projectDir: '/tmp/project',
|
||||
connectionId: 'warehouse',
|
||||
mode: 'structural',
|
||||
detectRelationships: false,
|
||||
dryRun: true,
|
||||
databaseIntrospectionUrl: undefined,
|
||||
},
|
||||
scanIo.io,
|
||||
);
|
||||
expect(scanIo.stderr()).toBe('');
|
||||
});
|
||||
|
||||
it('dispatches dev scan --mode relationships through Commander', async () => {
|
||||
const io = makeIo();
|
||||
const scan = vi.fn(async () => 0);
|
||||
|
||||
await expect(
|
||||
runKloCli(['dev', 'scan', 'warehouse', '--project-dir', '/tmp/project', '--mode', 'relationships'], io.io, {
|
||||
scan,
|
||||
}),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(scan).toHaveBeenCalledWith(
|
||||
{
|
||||
command: 'run',
|
||||
projectDir: '/tmp/project',
|
||||
connectionId: 'warehouse',
|
||||
mode: 'relationships',
|
||||
detectRelationships: true,
|
||||
dryRun: false,
|
||||
databaseIntrospectionUrl: undefined,
|
||||
},
|
||||
io.io,
|
||||
);
|
||||
expect(io.stderr()).toBe('');
|
||||
});
|
||||
|
||||
it.each(['--enrich', '--detect-relationships'])('rejects removed scan shorthand option %s', async (option) => {
|
||||
const io = makeIo();
|
||||
const scan = vi.fn(async () => 0);
|
||||
|
||||
await expect(runKloCli(['dev', 'scan', 'warehouse', option], io.io, { scan })).resolves.toBe(1);
|
||||
|
||||
expect(scan).not.toHaveBeenCalled();
|
||||
expect(io.stderr()).toContain(`unknown option '${option}'`);
|
||||
});
|
||||
|
||||
it('rejects dev scan without a connection id or subcommand', async () => {
|
||||
const io = makeIo();
|
||||
const scan = vi.fn(async () => 0);
|
||||
|
||||
await expect(runKloCli(['dev', 'scan', '--dry-run'], io.io, { scan })).resolves.toBe(1);
|
||||
|
||||
expect(scan).not.toHaveBeenCalled();
|
||||
expect(io.stdout()).toContain('Usage: klo dev scan');
|
||||
expect(io.stderr()).toContain('klo dev scan requires <connectionId> or a subcommand');
|
||||
});
|
||||
|
||||
it('rejects invalid scan modes before dispatch', async () => {
|
||||
const io = makeIo();
|
||||
const scan = vi.fn(async () => 0);
|
||||
|
||||
await expect(runKloCli(['dev', 'scan', 'warehouse', '--mode', 'deep'], io.io, { scan })).resolves.toBe(1);
|
||||
|
||||
expect(scan).not.toHaveBeenCalled();
|
||||
expect(io.stderr()).toContain("argument 'deep' is invalid");
|
||||
expect(io.stderr()).toContain('Allowed choices are structural, enriched, relationships');
|
||||
});
|
||||
|
||||
it('prints dev scan subcommand help with the canonical command name', async () => {
|
||||
const io = makeIo();
|
||||
const scan = vi.fn(async () => 0);
|
||||
|
||||
await expect(runKloCli(['dev', 'scan', 'report', '--help'], io.io, { scan })).resolves.toBe(0);
|
||||
|
||||
expect(io.stdout()).toContain('--project-dir is inherited from `klo dev scan`');
|
||||
expect(io.stdout()).not.toContain('--project-dir is inherited from `klo scan`');
|
||||
expect(scan).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('dispatches dev scan report in human and json modes', async () => {
|
||||
const humanIo = makeIo();
|
||||
const jsonIo = makeIo();
|
||||
const scan = vi.fn(async () => 0);
|
||||
|
||||
await expect(
|
||||
runKloCli(['dev', 'scan', 'report', 'scan-run-1', '--project-dir', '/tmp/project'], humanIo.io, { scan }),
|
||||
).resolves.toBe(0);
|
||||
await expect(
|
||||
runKloCli(['dev', 'scan', 'report', 'scan-run-2', '--project-dir', '/tmp/project', '--json'], jsonIo.io, {
|
||||
scan,
|
||||
}),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(scan).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
{ command: 'report', projectDir: '/tmp/project', runId: 'scan-run-1', json: false },
|
||||
humanIo.io,
|
||||
);
|
||||
expect(scan).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
{ command: 'report', projectDir: '/tmp/project', runId: 'scan-run-2', json: true },
|
||||
jsonIo.io,
|
||||
);
|
||||
});
|
||||
|
||||
it('dispatches dev scan relationships with filters through Commander', async () => {
|
||||
const io = makeIo();
|
||||
const scan = vi.fn(async () => 0);
|
||||
|
||||
await expect(
|
||||
runKloCli(
|
||||
[
|
||||
'dev',
|
||||
'scan',
|
||||
'relationships',
|
||||
'scan-run-review',
|
||||
'--project-dir',
|
||||
'/tmp/project',
|
||||
'--status',
|
||||
'rejected',
|
||||
'--limit',
|
||||
'5',
|
||||
'--json',
|
||||
],
|
||||
io.io,
|
||||
{ scan },
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(scan).toHaveBeenCalledWith(
|
||||
{
|
||||
command: 'relationships',
|
||||
projectDir: '/tmp/project',
|
||||
runId: 'scan-run-review',
|
||||
status: 'rejected',
|
||||
json: true,
|
||||
limit: 5,
|
||||
},
|
||||
io.io,
|
||||
);
|
||||
expect(io.stderr()).toBe('');
|
||||
});
|
||||
|
||||
it('dispatches dev scan relationship decision recording through Commander', async () => {
|
||||
const io = makeIo();
|
||||
const scan = vi.fn(async () => 0);
|
||||
|
||||
await expect(
|
||||
runKloCli(
|
||||
[
|
||||
'dev',
|
||||
'scan',
|
||||
'relationships',
|
||||
'scan-run-review',
|
||||
'--project-dir',
|
||||
'/tmp/project',
|
||||
'--accept',
|
||||
'orders:orders.customer_id->customers:customers.id',
|
||||
'--reviewer',
|
||||
'Andrey',
|
||||
'--note',
|
||||
'Looks right',
|
||||
'--json',
|
||||
],
|
||||
io.io,
|
||||
{ scan },
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(scan).toHaveBeenCalledWith(
|
||||
{
|
||||
command: 'relationshipDecision',
|
||||
projectDir: '/tmp/project',
|
||||
runId: 'scan-run-review',
|
||||
candidateId: 'orders:orders.customer_id->customers:customers.id',
|
||||
decision: 'accepted',
|
||||
reviewer: 'Andrey',
|
||||
note: 'Looks right',
|
||||
json: true,
|
||||
},
|
||||
io.io,
|
||||
);
|
||||
expect(io.stderr()).toBe('');
|
||||
});
|
||||
|
||||
it.each(['--accept', '--reject'])('rejects empty relationship decision candidate ids for %s', async (option) => {
|
||||
const io = makeIo();
|
||||
const scan = vi.fn(async () => 0);
|
||||
|
||||
await expect(
|
||||
runKloCli(['dev', 'scan', 'relationships', 'scan-run-review', option, ''], io.io, { scan }),
|
||||
).resolves.toBe(1);
|
||||
|
||||
expect(scan).not.toHaveBeenCalled();
|
||||
expect(io.stderr()).toContain('must not be empty');
|
||||
});
|
||||
|
||||
it('rejects relationship feedback JSON and JSONL output together', async () => {
|
||||
const io = makeIo();
|
||||
const scan = vi.fn(async () => 0);
|
||||
|
||||
await expect(
|
||||
runKloCli(['dev', 'scan', 'relationship-feedback', '--json', '--jsonl'], io.io, { scan }),
|
||||
).resolves.toBe(1);
|
||||
|
||||
expect(scan).not.toHaveBeenCalled();
|
||||
expect(io.stderr()).toMatch(/conflict|cannot be used/i);
|
||||
});
|
||||
|
||||
it('dispatches relationship apply command args', async () => {
|
||||
const io = makeIo();
|
||||
const scan = vi.fn(async () => 0);
|
||||
|
||||
await expect(
|
||||
runKloCli(
|
||||
[
|
||||
'dev',
|
||||
'scan',
|
||||
'relationship-apply',
|
||||
'scan-run-a',
|
||||
'--project-dir',
|
||||
'/tmp/project',
|
||||
'--candidate',
|
||||
'orders:orders.customer_id->customers:customers.id',
|
||||
'--dry-run',
|
||||
'--json',
|
||||
],
|
||||
io.io,
|
||||
{ scan },
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(scan).toHaveBeenCalledWith(
|
||||
{
|
||||
command: 'relationshipApply',
|
||||
projectDir: '/tmp/project',
|
||||
runId: 'scan-run-a',
|
||||
applyAllAccepted: false,
|
||||
candidateIds: ['orders:orders.customer_id->customers:customers.id'],
|
||||
dryRun: true,
|
||||
json: true,
|
||||
},
|
||||
io.io,
|
||||
);
|
||||
});
|
||||
|
||||
it('dispatches scan relationship feedback command with filters and JSONL output', async () => {
|
||||
const io = makeIo();
|
||||
const scan = vi.fn(async () => 0);
|
||||
|
||||
await expect(
|
||||
runKloCli(
|
||||
[
|
||||
'dev',
|
||||
'scan',
|
||||
'relationship-feedback',
|
||||
'--project-dir',
|
||||
'/tmp/project',
|
||||
'--connection',
|
||||
'warehouse',
|
||||
'--decision',
|
||||
'accepted',
|
||||
'--jsonl',
|
||||
],
|
||||
io.io,
|
||||
{ scan },
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(scan).toHaveBeenCalledWith(
|
||||
{
|
||||
command: 'relationshipFeedback',
|
||||
projectDir: '/tmp/project',
|
||||
connectionId: 'warehouse',
|
||||
decision: 'accepted',
|
||||
json: false,
|
||||
jsonl: true,
|
||||
},
|
||||
io.io,
|
||||
);
|
||||
});
|
||||
|
||||
it('dispatches scan relationship calibration command with thresholds', async () => {
|
||||
const io = makeIo();
|
||||
const scan = vi.fn(async () => 0);
|
||||
|
||||
await expect(
|
||||
runKloCli(
|
||||
[
|
||||
'dev',
|
||||
'scan',
|
||||
'relationship-calibration',
|
||||
'--project-dir',
|
||||
'/tmp/project',
|
||||
'--connection',
|
||||
'warehouse',
|
||||
'--decision',
|
||||
'rejected',
|
||||
'--accept-threshold',
|
||||
'0.9',
|
||||
'--review-threshold',
|
||||
'0.5',
|
||||
'--json',
|
||||
],
|
||||
io.io,
|
||||
{ scan },
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(scan).toHaveBeenCalledWith(
|
||||
{
|
||||
command: 'relationshipCalibration',
|
||||
projectDir: '/tmp/project',
|
||||
connectionId: 'warehouse',
|
||||
decision: 'rejected',
|
||||
acceptThreshold: 0.9,
|
||||
reviewThreshold: 0.5,
|
||||
json: true,
|
||||
},
|
||||
io.io,
|
||||
);
|
||||
});
|
||||
|
||||
it('dispatches relationship threshold advice command args', async () => {
|
||||
const io = makeIo();
|
||||
const scan = vi.fn(async () => 0);
|
||||
|
||||
await expect(
|
||||
runKloCli(
|
||||
[
|
||||
'dev',
|
||||
'scan',
|
||||
'relationship-thresholds',
|
||||
'--project-dir',
|
||||
'/tmp/project',
|
||||
'--connection',
|
||||
'warehouse',
|
||||
'--min-total-labels',
|
||||
'12',
|
||||
'--min-accepted-labels',
|
||||
'4',
|
||||
'--min-rejected-labels',
|
||||
'3',
|
||||
'--json',
|
||||
],
|
||||
io.io,
|
||||
{ scan },
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(scan).toHaveBeenCalledWith(
|
||||
{
|
||||
command: 'relationshipThresholds',
|
||||
projectDir: '/tmp/project',
|
||||
connectionId: 'warehouse',
|
||||
minTotalLabels: 12,
|
||||
minAcceptedLabels: 4,
|
||||
minRejectedLabels: 3,
|
||||
json: true,
|
||||
},
|
||||
io.io,
|
||||
);
|
||||
});
|
||||
|
||||
it('rejects invalid relationship calibration thresholds before dispatch', async () => {
|
||||
const io = makeIo();
|
||||
const scan = vi.fn(async () => 0);
|
||||
|
||||
await expect(
|
||||
runKloCli(['dev', 'scan', 'relationship-calibration', '--accept-threshold', '1.5'], io.io, { scan }),
|
||||
).resolves.toBe(1);
|
||||
|
||||
expect(scan).not.toHaveBeenCalled();
|
||||
expect(io.stderr()).toContain('Allowed range is 0 through 1');
|
||||
});
|
||||
|
||||
it('rejects relationship accept and reject options together before dispatch', async () => {
|
||||
const io = makeIo();
|
||||
const scan = vi.fn(async () => 0);
|
||||
|
||||
await expect(
|
||||
runKloCli(
|
||||
[
|
||||
'dev',
|
||||
'scan',
|
||||
'relationships',
|
||||
'scan-run-review',
|
||||
'--accept',
|
||||
'orders:orders.customer_id->customers:customers.id',
|
||||
'--reject',
|
||||
'orders:orders.customer_id->customers:customers.id',
|
||||
],
|
||||
io.io,
|
||||
{ scan },
|
||||
),
|
||||
).resolves.toBe(1);
|
||||
|
||||
expect(scan).not.toHaveBeenCalled();
|
||||
expect(io.stderr()).toMatch(/conflict|cannot be used/i);
|
||||
});
|
||||
|
||||
it('dispatches dev ingest run through the low-level ingest Commander registration', async () => {
|
||||
const io = makeIo();
|
||||
const ingest = vi.fn(async () => 0);
|
||||
|
||||
await expect(
|
||||
runKloCli(
|
||||
[
|
||||
'dev',
|
||||
'ingest',
|
||||
'run',
|
||||
'--connection-id',
|
||||
'warehouse',
|
||||
'--adapter',
|
||||
'metabase',
|
||||
'--project-dir',
|
||||
'/tmp/project',
|
||||
'--json',
|
||||
],
|
||||
io.io,
|
||||
{ ingest },
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(ingest).toHaveBeenCalledWith(
|
||||
{
|
||||
command: 'run',
|
||||
projectDir: '/tmp/project',
|
||||
connectionId: 'warehouse',
|
||||
adapter: 'metabase',
|
||||
sourceDir: undefined,
|
||||
databaseIntrospectionUrl: undefined,
|
||||
outputMode: 'json',
|
||||
},
|
||||
io.io,
|
||||
);
|
||||
expect(io.stderr()).toBe('');
|
||||
});
|
||||
});
|
||||
61
packages/cli/src/dev.ts
Normal file
61
packages/cli/src/dev.ts
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
import { resolve } from 'node:path';
|
||||
import type { Command } from '@commander-js/extra-typings';
|
||||
import { type CommandWithGlobalOptions, type KloCliCommandContext, resolveCommandProjectDir } from './cli-program.js';
|
||||
import { registerCompletionCommands } from './commands/completion-commands.js';
|
||||
import { registerConnectionMappingCommands } from './commands/connection-commands.js';
|
||||
import { registerDoctorCommands } from './commands/doctor-commands.js';
|
||||
import { registerIngestCommands } from './commands/ingest-commands.js';
|
||||
import { registerScanCommands } from './commands/scan-commands.js';
|
||||
import { profileMark } from './startup-profile.js';
|
||||
|
||||
profileMark('module:dev');
|
||||
|
||||
export function registerDevCommands(program: Command, context: KloCliCommandContext): void {
|
||||
const dev = program
|
||||
.command('dev', { hidden: true })
|
||||
.description('Low-level diagnostics, scans, adapter commands, and mapping tools')
|
||||
.showHelpAfterError();
|
||||
|
||||
dev.hook('preAction', (_thisCommand, actionCommand) => {
|
||||
context.writeDebug?.('dev', actionCommand);
|
||||
});
|
||||
|
||||
dev.action(() => {
|
||||
dev.outputHelp();
|
||||
context.setExitCode(0);
|
||||
});
|
||||
|
||||
dev
|
||||
.command('init')
|
||||
.description('Initialize a Git-backed KLO project directory for maintenance scripts')
|
||||
.argument('[directory]', 'Project directory')
|
||||
.option('--name <name>', 'Project name written to klo.yaml')
|
||||
.option('--force', 'Rewrite klo.yaml and scaffold files in an existing project', false)
|
||||
.action(
|
||||
async (
|
||||
projectDir: string | undefined,
|
||||
commandOptions: { name?: string; force?: boolean },
|
||||
command: CommandWithGlobalOptions,
|
||||
) => {
|
||||
context.setExitCode(
|
||||
await context.runInit(
|
||||
{
|
||||
projectDir: projectDir ? resolve(projectDir) : resolveCommandProjectDir(command),
|
||||
...(commandOptions.name ? { projectName: commandOptions.name } : {}),
|
||||
force: commandOptions.force === true,
|
||||
},
|
||||
context.io,
|
||||
),
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
registerDoctorCommands(dev, context);
|
||||
registerScanCommands(dev, context);
|
||||
registerIngestCommands(dev, context, {
|
||||
runIngestWithProgress: async (ingestArgs, ingestIo, ingestDeps, defaultRunIngest) =>
|
||||
await (ingestDeps.ingest ?? defaultRunIngest)(ingestArgs, ingestIo),
|
||||
});
|
||||
registerConnectionMappingCommands(dev, context);
|
||||
registerCompletionCommands(dev, context, program);
|
||||
}
|
||||
460
packages/cli/src/doctor.test.ts
Normal file
460
packages/cli/src/doctor.test.ts
Normal file
|
|
@ -0,0 +1,460 @@
|
|||
import { mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import type { KloEmbeddingConfig, KloEmbeddingHealthCheckOptions, KloEmbeddingHealthCheckResult } from '@klo/llm';
|
||||
import {
|
||||
formatDoctorReport,
|
||||
runKloDoctor,
|
||||
runSetupDoctorChecks,
|
||||
type DoctorCheck,
|
||||
} from './doctor.js';
|
||||
|
||||
function makeIo() {
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
return {
|
||||
io: {
|
||||
stdout: {
|
||||
write: (chunk: string) => {
|
||||
stdout += chunk;
|
||||
},
|
||||
},
|
||||
stderr: {
|
||||
write: (chunk: string) => {
|
||||
stderr += chunk;
|
||||
},
|
||||
},
|
||||
},
|
||||
stdout: () => stdout,
|
||||
stderr: () => stderr,
|
||||
};
|
||||
}
|
||||
|
||||
type EmbeddingHealthCheck = (
|
||||
config: KloEmbeddingConfig,
|
||||
options?: KloEmbeddingHealthCheckOptions,
|
||||
) => Promise<KloEmbeddingHealthCheckResult>;
|
||||
|
||||
async function writeProjectConfig(projectDir: string, embeddingLines: string[]): Promise<void> {
|
||||
await writeFile(
|
||||
join(projectDir, 'klo.yaml'),
|
||||
[
|
||||
'project: warehouse',
|
||||
'connections:',
|
||||
' warehouse:',
|
||||
' driver: sqlite',
|
||||
' path: ./warehouse.db',
|
||||
'ingest:',
|
||||
' adapters:',
|
||||
' - live-database',
|
||||
' embeddings:',
|
||||
...embeddingLines.map((line) => ` ${line}`),
|
||||
'',
|
||||
].join('\n'),
|
||||
'utf-8',
|
||||
);
|
||||
}
|
||||
|
||||
describe('formatDoctorReport', () => {
|
||||
it('prints exact fixes for failing setup checks', () => {
|
||||
const checks: DoctorCheck[] = [
|
||||
{ id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0 ABI 127' },
|
||||
{
|
||||
id: 'native-sqlite',
|
||||
label: 'Native SQLite',
|
||||
status: 'fail',
|
||||
detail: 'Cannot load better-sqlite3',
|
||||
fix: 'Run: pnpm run native:rebuild',
|
||||
},
|
||||
];
|
||||
|
||||
expect(formatDoctorReport({ title: 'KLO setup doctor', checks })).toBe(
|
||||
[
|
||||
'KLO setup doctor',
|
||||
'PASS Node 22+: v22.16.0 ABI 127',
|
||||
'FAIL Native SQLite: Cannot load better-sqlite3',
|
||||
' Fix: Run: pnpm run native:rebuild',
|
||||
'',
|
||||
].join('\n'),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('runSetupDoctorChecks', () => {
|
||||
it('returns pass checks when injected commands and file checks succeed', async () => {
|
||||
const checks = await runSetupDoctorChecks({
|
||||
env: { PATH: '/bin' },
|
||||
workspaceRoot: '/workspace/klo',
|
||||
execText: async (command, args) => {
|
||||
if (command === 'pnpm' && args[0] === '--version') return '10.28.0';
|
||||
if (command === 'corepack' && args[0] === '--version') return '0.32.0';
|
||||
if (command === 'uv' && args[0] === '--version') return 'uv 0.9.5';
|
||||
if (command === process.execPath && args.includes('--version')) return '@klo/cli 0.0.0-private';
|
||||
throw new Error(`${command} ${args.join(' ')}`);
|
||||
},
|
||||
pathExists: async () => true,
|
||||
importBetterSqlite3: async () => ({ default: function Database() {} }),
|
||||
});
|
||||
|
||||
expect(checks.map((check) => [check.id, check.status])).toEqual([
|
||||
['node', 'pass'],
|
||||
['pnpm', 'pass'],
|
||||
['corepack', 'pass'],
|
||||
['uv', 'pass'],
|
||||
['native-sqlite', 'pass'],
|
||||
['package-build', 'pass'],
|
||||
['workspace-cli', 'pass'],
|
||||
]);
|
||||
});
|
||||
|
||||
it('returns exact fixes when setup checks fail', async () => {
|
||||
const checks = await runSetupDoctorChecks({
|
||||
env: {},
|
||||
workspaceRoot: '/workspace/klo',
|
||||
execText: async (command) => {
|
||||
throw new Error(`${command} not found`);
|
||||
},
|
||||
pathExists: async () => false,
|
||||
importBetterSqlite3: async () => {
|
||||
throw new Error('Cannot find module better-sqlite3');
|
||||
},
|
||||
});
|
||||
|
||||
expect(checks).toContainEqual({
|
||||
id: 'pnpm',
|
||||
label: 'pnpm 10.20+',
|
||||
status: 'fail',
|
||||
detail: 'pnpm not found',
|
||||
fix: 'Run: corepack enable && corepack prepare pnpm@10.28.0 --activate',
|
||||
});
|
||||
expect(checks).toContainEqual({
|
||||
id: 'package-build',
|
||||
label: 'TypeScript package build',
|
||||
status: 'fail',
|
||||
detail: 'Missing packages/cli/dist/bin.js',
|
||||
fix: 'Run: pnpm run build',
|
||||
});
|
||||
});
|
||||
|
||||
it('treats missing corepack as a warning so setup doctor can still pass', async () => {
|
||||
const checks = await runSetupDoctorChecks({
|
||||
env: { PATH: '/bin' },
|
||||
workspaceRoot: '/workspace/klo',
|
||||
execText: async (command, args) => {
|
||||
if (command === 'pnpm' && args[0] === '--version') return '10.28.0';
|
||||
if (command === 'corepack' && args[0] === '--version') throw new Error('spawn corepack ENOENT');
|
||||
if (command === 'uv' && args[0] === '--version') return 'uv 0.9.5';
|
||||
if (command === process.execPath && args.includes('--version')) return '@klo/cli 0.0.0-private';
|
||||
throw new Error(`${command} ${args.join(' ')}`);
|
||||
},
|
||||
pathExists: async () => true,
|
||||
importBetterSqlite3: async () => ({ default: function Database() {} }),
|
||||
});
|
||||
const testIo = makeIo();
|
||||
|
||||
await expect(
|
||||
runKloDoctor({ command: 'setup', outputMode: 'plain', inputMode: 'disabled' }, testIo.io, {
|
||||
runSetupChecks: async () => checks,
|
||||
}),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(checks).toContainEqual({
|
||||
id: 'corepack',
|
||||
label: 'Corepack',
|
||||
status: 'warn',
|
||||
detail: 'spawn corepack ENOENT',
|
||||
fix: 'Run: corepack enable',
|
||||
});
|
||||
expect(testIo.stdout()).toContain('WARN Corepack: spawn corepack ENOENT');
|
||||
expect(testIo.stderr()).toBe('');
|
||||
});
|
||||
});
|
||||
|
||||
describe('runKloDoctor', () => {
|
||||
let tempDir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'klo-doctor-'));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(tempDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('prints setup report and exits nonzero when a check fails', async () => {
|
||||
const testIo = makeIo();
|
||||
|
||||
await expect(
|
||||
runKloDoctor(
|
||||
{ command: 'setup', outputMode: 'plain', inputMode: 'disabled' },
|
||||
testIo.io,
|
||||
{
|
||||
runSetupChecks: async () => [
|
||||
{ id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0 ABI 127' },
|
||||
{
|
||||
id: 'package-build',
|
||||
label: 'TypeScript package build',
|
||||
status: 'fail',
|
||||
detail: 'Missing packages/cli/dist/bin.js',
|
||||
fix: 'Run: pnpm run build',
|
||||
},
|
||||
],
|
||||
},
|
||||
),
|
||||
).resolves.toBe(1);
|
||||
|
||||
expect(testIo.stdout()).toContain('KLO setup doctor');
|
||||
expect(testIo.stdout()).toContain('FAIL TypeScript package build: Missing packages/cli/dist/bin.js');
|
||||
expect(testIo.stdout()).toContain('Fix: Run: pnpm run build');
|
||||
expect(testIo.stderr()).toBe('');
|
||||
});
|
||||
|
||||
it('prints JSON setup report', async () => {
|
||||
const testIo = makeIo();
|
||||
|
||||
await expect(
|
||||
runKloDoctor(
|
||||
{ command: 'setup', outputMode: 'json', inputMode: 'disabled' },
|
||||
testIo.io,
|
||||
{
|
||||
runSetupChecks: async () => [
|
||||
{ id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0 ABI 127' },
|
||||
],
|
||||
},
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(JSON.parse(testIo.stdout())).toEqual({
|
||||
title: 'KLO setup doctor',
|
||||
checks: [{ id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0 ABI 127' }],
|
||||
});
|
||||
});
|
||||
|
||||
it('runs project checks against a valid klo.yaml', async () => {
|
||||
await writeFile(
|
||||
join(tempDir, 'klo.yaml'),
|
||||
[
|
||||
'project: warehouse',
|
||||
'connections:',
|
||||
' warehouse:',
|
||||
' driver: sqlite',
|
||||
' path: ./warehouse.db',
|
||||
'ingest:',
|
||||
' adapters:',
|
||||
' - live-database',
|
||||
'',
|
||||
].join('\n'),
|
||||
'utf-8',
|
||||
);
|
||||
const testIo = makeIo();
|
||||
|
||||
await expect(
|
||||
runKloDoctor(
|
||||
{ command: 'project', projectDir: tempDir, outputMode: 'plain', inputMode: 'disabled' },
|
||||
testIo.io,
|
||||
{
|
||||
runSetupChecks: async () => [
|
||||
{ id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0 ABI 127' },
|
||||
],
|
||||
},
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(testIo.stdout()).toContain('KLO project doctor');
|
||||
expect(testIo.stdout()).toContain('PASS Project config: warehouse');
|
||||
expect(testIo.stdout()).toContain('PASS Connections: 1 configured');
|
||||
});
|
||||
|
||||
it('includes Postgres historic-SQL readiness in project doctor output', async () => {
|
||||
await writeFile(
|
||||
join(tempDir, 'klo.yaml'),
|
||||
[
|
||||
'project: warehouse',
|
||||
'connections:',
|
||||
' warehouse:',
|
||||
' driver: postgres',
|
||||
' url: env:WAREHOUSE_DATABASE_URL',
|
||||
' readonly: true',
|
||||
' historicSql:',
|
||||
' enabled: true',
|
||||
' dialect: postgres',
|
||||
'ingest:',
|
||||
' adapters:',
|
||||
' - live-database',
|
||||
' - historic-sql',
|
||||
'',
|
||||
].join('\n'),
|
||||
'utf-8',
|
||||
);
|
||||
const testIo = makeIo();
|
||||
const runHistoricSqlDoctorChecks = vi.fn(async () => [
|
||||
{
|
||||
id: 'historic-sql-postgres-warehouse',
|
||||
label: 'Postgres Historic SQL (warehouse)',
|
||||
status: 'warn' as const,
|
||||
detail:
|
||||
'pg_stat_statements ready (PostgreSQL 16.4) with warnings: pg_stat_statements.max is 1000; set it to at least 5000 to reduce query-template eviction churn',
|
||||
fix: `Update the Postgres parameter group or config, then rerun \`klo dev doctor --project-dir ${tempDir}\``,
|
||||
},
|
||||
]);
|
||||
|
||||
await expect(
|
||||
runKloDoctor(
|
||||
{ command: 'project', projectDir: tempDir, outputMode: 'plain', inputMode: 'disabled' },
|
||||
testIo.io,
|
||||
{
|
||||
runSetupChecks: async () => [
|
||||
{ id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0 ABI 127' },
|
||||
],
|
||||
runHistoricSqlDoctorChecks,
|
||||
},
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(runHistoricSqlDoctorChecks).toHaveBeenCalledTimes(1);
|
||||
expect(testIo.stdout()).toContain('WARN Postgres Historic SQL (warehouse): pg_stat_statements ready');
|
||||
expect(testIo.stdout()).toContain('Fix: Update the Postgres parameter group or config');
|
||||
});
|
||||
|
||||
it('warns when semantic-search embeddings are not configured', async () => {
|
||||
await writeProjectConfig(tempDir, ['backend: deterministic', 'model: deterministic', 'dimensions: 8']);
|
||||
const testIo = makeIo();
|
||||
|
||||
await expect(
|
||||
runKloDoctor(
|
||||
{ command: 'project', projectDir: tempDir, outputMode: 'plain', inputMode: 'disabled' },
|
||||
testIo.io,
|
||||
{
|
||||
runSetupChecks: async () => [
|
||||
{ id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0 ABI 127' },
|
||||
],
|
||||
},
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(testIo.stdout()).toContain('WARN Semantic search embeddings: ingest.embeddings.backend is deterministic.');
|
||||
expect(testIo.stdout()).toContain(
|
||||
'Semantic lane will be skipped; lexical, dictionary, and token lanes remain available.',
|
||||
);
|
||||
expect(testIo.stdout()).toContain(
|
||||
`Fix: Run: klo setup --project-dir ${tempDir} --no-input`,
|
||||
);
|
||||
});
|
||||
|
||||
it('probes configured semantic-search embeddings for project doctor', async () => {
|
||||
await writeProjectConfig(tempDir, [
|
||||
'backend: sentence-transformers',
|
||||
'model: all-MiniLM-L6-v2',
|
||||
'dimensions: 384',
|
||||
'sentenceTransformers:',
|
||||
' base_url: http://127.0.0.1:8765',
|
||||
" pathPrefix: ''",
|
||||
]);
|
||||
const healthCheck = vi.fn<EmbeddingHealthCheck>(async () => ({ ok: true }));
|
||||
const testIo = makeIo();
|
||||
|
||||
await expect(
|
||||
runKloDoctor(
|
||||
{ command: 'project', projectDir: tempDir, outputMode: 'plain', inputMode: 'disabled' },
|
||||
testIo.io,
|
||||
{
|
||||
runSetupChecks: async () => [
|
||||
{ id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0 ABI 127' },
|
||||
],
|
||||
embeddingHealthCheck: healthCheck,
|
||||
embeddingProbeTimeoutMs: 1234,
|
||||
},
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(healthCheck).toHaveBeenCalledWith(
|
||||
{
|
||||
backend: 'sentence-transformers',
|
||||
model: 'all-MiniLM-L6-v2',
|
||||
dimensions: 384,
|
||||
sentenceTransformers: { baseURL: 'http://127.0.0.1:8765', pathPrefix: '' },
|
||||
},
|
||||
{ text: 'KLO semantic search doctor probe', timeoutMs: 1234 },
|
||||
);
|
||||
expect(testIo.stdout()).toContain(
|
||||
'PASS Semantic search embeddings: sentence-transformers/all-MiniLM-L6-v2 (384d) probe succeeded',
|
||||
);
|
||||
});
|
||||
|
||||
it('allows local sentence-transformers semantic-search probes enough time for cold start', async () => {
|
||||
await writeProjectConfig(tempDir, [
|
||||
'backend: sentence-transformers',
|
||||
'model: all-MiniLM-L6-v2',
|
||||
'dimensions: 384',
|
||||
'sentenceTransformers:',
|
||||
' base_url: http://127.0.0.1:8765',
|
||||
" pathPrefix: ''",
|
||||
]);
|
||||
const healthCheck = vi.fn<EmbeddingHealthCheck>(async () => ({ ok: true }));
|
||||
const testIo = makeIo();
|
||||
|
||||
await expect(
|
||||
runKloDoctor(
|
||||
{ command: 'project', projectDir: tempDir, outputMode: 'plain', inputMode: 'disabled' },
|
||||
testIo.io,
|
||||
{
|
||||
runSetupChecks: async () => [
|
||||
{ id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0 ABI 127' },
|
||||
],
|
||||
embeddingHealthCheck: healthCheck,
|
||||
},
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(healthCheck).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
backend: 'sentence-transformers',
|
||||
model: 'all-MiniLM-L6-v2',
|
||||
dimensions: 384,
|
||||
}),
|
||||
{ text: 'KLO semantic search doctor probe', timeoutMs: 120_000 },
|
||||
);
|
||||
});
|
||||
|
||||
it('reports unhealthy semantic-search embeddings as a warning in JSON output', async () => {
|
||||
await writeProjectConfig(tempDir, [
|
||||
'backend: sentence-transformers',
|
||||
'model: all-MiniLM-L6-v2',
|
||||
'dimensions: 384',
|
||||
'sentenceTransformers:',
|
||||
' base_url: http://127.0.0.1:8765',
|
||||
" pathPrefix: ''",
|
||||
]);
|
||||
const healthCheck = vi.fn<EmbeddingHealthCheck>(async () => ({
|
||||
ok: false,
|
||||
message: 'connect ECONNREFUSED 127.0.0.1:8765',
|
||||
}));
|
||||
const testIo = makeIo();
|
||||
|
||||
await expect(
|
||||
runKloDoctor(
|
||||
{ command: 'project', projectDir: tempDir, outputMode: 'json', inputMode: 'disabled' },
|
||||
testIo.io,
|
||||
{
|
||||
runSetupChecks: async () => [
|
||||
{ id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0 ABI 127' },
|
||||
],
|
||||
embeddingHealthCheck: healthCheck,
|
||||
},
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
const report = JSON.parse(testIo.stdout()) as {
|
||||
checks: Array<{ id: string; label: string; status: string; detail: string; fix?: string }>;
|
||||
};
|
||||
expect(report.checks).toContainEqual({
|
||||
id: 'semantic-search-embeddings',
|
||||
label: 'Semantic search embeddings',
|
||||
status: 'warn',
|
||||
detail:
|
||||
'sentence-transformers/all-MiniLM-L6-v2 (384d) probe failed: connect ECONNREFUSED 127.0.0.1:8765. Semantic lane will be skipped; lexical, dictionary, and token lanes remain available.',
|
||||
fix: `Run: klo setup --project-dir ${tempDir} --no-input`,
|
||||
});
|
||||
});
|
||||
});
|
||||
488
packages/cli/src/doctor.ts
Normal file
488
packages/cli/src/doctor.ts
Normal file
|
|
@ -0,0 +1,488 @@
|
|||
import { execFile } from 'node:child_process';
|
||||
import { constants as fsConstants } from 'node:fs';
|
||||
import { access } from 'node:fs/promises';
|
||||
import { join, resolve } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { promisify } from 'node:util';
|
||||
import type { KloLocalProject, KloProjectEmbeddingConfig } from '@klo/context/project';
|
||||
import type { KloEmbeddingConfig, KloEmbeddingHealthCheckOptions, KloEmbeddingHealthCheckResult } from '@klo/llm';
|
||||
import type { HistoricSqlDoctorDeps } from './historic-sql-doctor.js';
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
type DoctorStatus = 'pass' | 'warn' | 'fail';
|
||||
type KloDoctorOutputMode = 'plain' | 'json';
|
||||
type KloDoctorInputMode = 'auto' | 'disabled';
|
||||
|
||||
export interface DoctorCheck {
|
||||
id: string;
|
||||
label: string;
|
||||
status: DoctorStatus;
|
||||
detail: string;
|
||||
fix?: string;
|
||||
}
|
||||
|
||||
interface DoctorReport {
|
||||
title: string;
|
||||
checks: DoctorCheck[];
|
||||
}
|
||||
|
||||
export type KloDoctorArgs =
|
||||
| { command: 'setup'; outputMode: KloDoctorOutputMode; inputMode?: KloDoctorInputMode }
|
||||
| { command: 'project'; projectDir: string; outputMode: KloDoctorOutputMode; inputMode?: KloDoctorInputMode }
|
||||
| { command: 'demo'; projectDir: string; outputMode: KloDoctorOutputMode; inputMode?: KloDoctorInputMode };
|
||||
|
||||
interface KloDoctorIo {
|
||||
stdout: { write(chunk: string): void };
|
||||
stderr: { write(chunk: string): void };
|
||||
}
|
||||
|
||||
interface SetupDoctorDeps {
|
||||
env?: NodeJS.ProcessEnv;
|
||||
workspaceRoot?: string;
|
||||
execText?: (command: string, args: string[], options?: { cwd?: string; env?: NodeJS.ProcessEnv }) => Promise<string>;
|
||||
pathExists?: (path: string) => Promise<boolean>;
|
||||
importBetterSqlite3?: () => Promise<unknown>;
|
||||
}
|
||||
|
||||
type EmbeddingHealthCheck = (
|
||||
config: KloEmbeddingConfig,
|
||||
options?: KloEmbeddingHealthCheckOptions,
|
||||
) => Promise<KloEmbeddingHealthCheckResult>;
|
||||
|
||||
interface SemanticSearchDoctorDeps {
|
||||
env?: NodeJS.ProcessEnv;
|
||||
embeddingHealthCheck?: EmbeddingHealthCheck;
|
||||
embeddingProbeTimeoutMs?: number;
|
||||
}
|
||||
|
||||
interface KloDoctorDeps extends SemanticSearchDoctorDeps, HistoricSqlDoctorDeps {
|
||||
runSetupChecks?: () => Promise<DoctorCheck[]>;
|
||||
runHistoricSqlDoctorChecks?: (project: KloLocalProject, deps: HistoricSqlDoctorDeps) => Promise<DoctorCheck[]>;
|
||||
}
|
||||
|
||||
function workspaceRootDir(): string {
|
||||
return resolve(fileURLToPath(new URL('../../../', import.meta.url)));
|
||||
}
|
||||
|
||||
async function defaultExecText(
|
||||
command: string,
|
||||
args: string[],
|
||||
options: { cwd?: string; env?: NodeJS.ProcessEnv } = {},
|
||||
): Promise<string> {
|
||||
const result = await execFileAsync(command, args, {
|
||||
cwd: options.cwd,
|
||||
env: options.env,
|
||||
encoding: 'utf8',
|
||||
maxBuffer: 1024 * 1024,
|
||||
});
|
||||
return `${result.stdout}${result.stderr}`.trim();
|
||||
}
|
||||
|
||||
async function defaultPathExists(path: string): Promise<boolean> {
|
||||
try {
|
||||
await access(path, fsConstants.F_OK);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function failureMessage(error: unknown): string {
|
||||
if (error instanceof Error && error.message.trim().length > 0) {
|
||||
return error.message.trim().split('\n')[0] ?? error.message.trim();
|
||||
}
|
||||
return String(error);
|
||||
}
|
||||
|
||||
function parseVersion(value: string): number[] {
|
||||
const match = value.match(/(\d+)\.(\d+)\.(\d+)/);
|
||||
if (!match) {
|
||||
return [];
|
||||
}
|
||||
return [Number(match[1]), Number(match[2]), Number(match[3])];
|
||||
}
|
||||
|
||||
function versionAtLeast(value: string, minimum: [number, number, number]): boolean {
|
||||
const parsed = parseVersion(value);
|
||||
if (parsed.length !== 3) {
|
||||
return false;
|
||||
}
|
||||
for (let index = 0; index < minimum.length; index += 1) {
|
||||
if (parsed[index] > minimum[index]) return true;
|
||||
if (parsed[index] < minimum[index]) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function check(status: DoctorStatus, id: string, label: string, detail: string, fix?: string): DoctorCheck {
|
||||
return fix ? { id, label, status, detail, fix } : { id, label, status, detail };
|
||||
}
|
||||
|
||||
const SEMANTIC_SEARCH_HEALTH_TEXT = 'KLO semantic search doctor probe';
|
||||
const SEMANTIC_SEARCH_HEALTH_TIMEOUT_MS = 5_000;
|
||||
const SEMANTIC_SEARCH_LOCAL_HEALTH_TIMEOUT_MS = 120_000;
|
||||
|
||||
function semanticEmbeddingSetupFix(projectDir: string, backend: KloProjectEmbeddingConfig['backend']): string {
|
||||
if (backend === 'openai') {
|
||||
return `Set OPENAI_API_KEY or rerun: klo setup --project-dir ${projectDir} --embedding-backend openai --no-input`;
|
||||
}
|
||||
return `Run: klo setup --project-dir ${projectDir} --no-input`;
|
||||
}
|
||||
|
||||
function embeddingConfigLabel(config: KloProjectEmbeddingConfig | KloEmbeddingConfig): string {
|
||||
const model = config.model?.trim() || 'model not configured';
|
||||
return `${config.backend}/${model} (${config.dimensions}d)`;
|
||||
}
|
||||
|
||||
function semanticLaneFallbackDetail(reason: string): string {
|
||||
return `${reason}. Semantic lane will be skipped; lexical, dictionary, and token lanes remain available.`;
|
||||
}
|
||||
|
||||
async function defaultEmbeddingHealthCheck(
|
||||
config: KloEmbeddingConfig,
|
||||
options?: KloEmbeddingHealthCheckOptions,
|
||||
): Promise<KloEmbeddingHealthCheckResult> {
|
||||
const { runKloEmbeddingHealthCheck } = await import('@klo/llm');
|
||||
return runKloEmbeddingHealthCheck(config, options);
|
||||
}
|
||||
|
||||
async function runSemanticSearchEmbeddingCheck(
|
||||
config: KloProjectEmbeddingConfig,
|
||||
projectDir: string,
|
||||
deps: SemanticSearchDoctorDeps = {},
|
||||
): Promise<DoctorCheck> {
|
||||
if (config.backend === 'none' || config.backend === 'deterministic') {
|
||||
return check(
|
||||
'warn',
|
||||
'semantic-search-embeddings',
|
||||
'Semantic search embeddings',
|
||||
semanticLaneFallbackDetail(`ingest.embeddings.backend is ${config.backend}`),
|
||||
semanticEmbeddingSetupFix(projectDir, config.backend),
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
const { resolveLocalKloEmbeddingConfig } = await import('@klo/context');
|
||||
const resolved = resolveLocalKloEmbeddingConfig(config, deps.env ?? process.env);
|
||||
if (!resolved) {
|
||||
return check(
|
||||
'warn',
|
||||
'semantic-search-embeddings',
|
||||
'Semantic search embeddings',
|
||||
semanticLaneFallbackDetail(`No runtime embedding config resolved for ${embeddingConfigLabel(config)}`),
|
||||
semanticEmbeddingSetupFix(projectDir, config.backend),
|
||||
);
|
||||
}
|
||||
|
||||
const healthCheck = deps.embeddingHealthCheck ?? defaultEmbeddingHealthCheck;
|
||||
const timeoutMs =
|
||||
deps.embeddingProbeTimeoutMs ??
|
||||
(resolved.backend === 'sentence-transformers'
|
||||
? SEMANTIC_SEARCH_LOCAL_HEALTH_TIMEOUT_MS
|
||||
: SEMANTIC_SEARCH_HEALTH_TIMEOUT_MS);
|
||||
const health = await healthCheck(resolved, {
|
||||
text: SEMANTIC_SEARCH_HEALTH_TEXT,
|
||||
timeoutMs,
|
||||
});
|
||||
if (health.ok) {
|
||||
return check(
|
||||
'pass',
|
||||
'semantic-search-embeddings',
|
||||
'Semantic search embeddings',
|
||||
`${embeddingConfigLabel(resolved)} probe succeeded`,
|
||||
);
|
||||
}
|
||||
|
||||
return check(
|
||||
'warn',
|
||||
'semantic-search-embeddings',
|
||||
'Semantic search embeddings',
|
||||
semanticLaneFallbackDetail(`${embeddingConfigLabel(resolved)} probe failed: ${health.message}`),
|
||||
semanticEmbeddingSetupFix(projectDir, config.backend),
|
||||
);
|
||||
} catch (error) {
|
||||
return check(
|
||||
'warn',
|
||||
'semantic-search-embeddings',
|
||||
'Semantic search embeddings',
|
||||
semanticLaneFallbackDetail(`${embeddingConfigLabel(config)} probe failed: ${failureMessage(error)}`),
|
||||
semanticEmbeddingSetupFix(projectDir, config.backend),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export async function runSetupDoctorChecks(deps: SetupDoctorDeps = {}): Promise<DoctorCheck[]> {
|
||||
const env = deps.env ?? process.env;
|
||||
const root = deps.workspaceRoot ?? workspaceRootDir();
|
||||
const execText = deps.execText ?? defaultExecText;
|
||||
const pathExists = deps.pathExists ?? defaultPathExists;
|
||||
const importBetterSqlite3 = deps.importBetterSqlite3 ?? (() => import('better-sqlite3'));
|
||||
const checks: DoctorCheck[] = [];
|
||||
|
||||
const nodeDetail = `${process.version} ABI ${process.versions.modules}`;
|
||||
checks.push(
|
||||
versionAtLeast(process.version, [22, 0, 0])
|
||||
? check('pass', 'node', 'Node 22+', nodeDetail)
|
||||
: check('fail', 'node', 'Node 22+', nodeDetail, 'Install Node 22 or newer, then rerun `pnpm run setup:dev`'),
|
||||
);
|
||||
|
||||
try {
|
||||
const pnpmVersion = await execText('pnpm', ['--version'], { cwd: root, env });
|
||||
checks.push(
|
||||
versionAtLeast(pnpmVersion, [10, 20, 0])
|
||||
? check('pass', 'pnpm', 'pnpm 10.20+', pnpmVersion)
|
||||
: check(
|
||||
'fail',
|
||||
'pnpm',
|
||||
'pnpm 10.20+',
|
||||
pnpmVersion,
|
||||
'Run: corepack enable && corepack prepare pnpm@10.28.0 --activate',
|
||||
),
|
||||
);
|
||||
} catch (error) {
|
||||
checks.push(
|
||||
check(
|
||||
'fail',
|
||||
'pnpm',
|
||||
'pnpm 10.20+',
|
||||
failureMessage(error),
|
||||
'Run: corepack enable && corepack prepare pnpm@10.28.0 --activate',
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
const corepackVersion = await execText('corepack', ['--version'], { cwd: root, env });
|
||||
checks.push(check('pass', 'corepack', 'Corepack', corepackVersion));
|
||||
} catch (error) {
|
||||
checks.push(check('warn', 'corepack', 'Corepack', failureMessage(error), 'Run: corepack enable'));
|
||||
}
|
||||
|
||||
try {
|
||||
const uvVersion = await execText('uv', ['--version'], { cwd: root, env });
|
||||
checks.push(check('pass', 'uv', 'uv', uvVersion));
|
||||
} catch (error) {
|
||||
checks.push(check('fail', 'uv', 'uv', failureMessage(error), 'Install uv, then rerun `pnpm run setup:dev`'));
|
||||
}
|
||||
|
||||
try {
|
||||
await importBetterSqlite3();
|
||||
checks.push(check('pass', 'native-sqlite', 'Native SQLite', 'better-sqlite3 loaded'));
|
||||
} catch (error) {
|
||||
checks.push(
|
||||
check('fail', 'native-sqlite', 'Native SQLite', failureMessage(error), 'Run: pnpm run native:rebuild'),
|
||||
);
|
||||
}
|
||||
|
||||
const cliBin = join(root, 'packages/cli/dist/bin.js');
|
||||
if (await pathExists(cliBin)) {
|
||||
checks.push(check('pass', 'package-build', 'TypeScript package build', 'packages/cli/dist/bin.js exists'));
|
||||
} else {
|
||||
checks.push(
|
||||
check(
|
||||
'fail',
|
||||
'package-build',
|
||||
'TypeScript package build',
|
||||
'Missing packages/cli/dist/bin.js',
|
||||
'Run: pnpm run build',
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
const output = await execText(process.execPath, [cliBin, '--version'], { cwd: root, env });
|
||||
checks.push(check('pass', 'workspace-cli', 'Workspace-local CLI', output));
|
||||
} catch (error) {
|
||||
checks.push(
|
||||
check(
|
||||
'fail',
|
||||
'workspace-cli',
|
||||
'Workspace-local CLI',
|
||||
failureMessage(error),
|
||||
'Run: pnpm run build && pnpm run klo -- --version',
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
return checks;
|
||||
}
|
||||
|
||||
async function runProjectChecks(projectDir: string, deps: KloDoctorDeps = {}): Promise<DoctorCheck[]> {
|
||||
const { loadKloProject } = await import('@klo/context/project');
|
||||
const checks: DoctorCheck[] = [];
|
||||
try {
|
||||
const project = await loadKloProject({ projectDir });
|
||||
checks.push(check('pass', 'project-config', 'Project config', project.config.project));
|
||||
const connectionCount = Object.keys(project.config.connections).length;
|
||||
checks.push(
|
||||
connectionCount > 0
|
||||
? check('pass', 'connections', 'Connections', `${connectionCount} configured`)
|
||||
: check(
|
||||
'warn',
|
||||
'connections',
|
||||
'Connections',
|
||||
'0 configured',
|
||||
'Add a connection to klo.yaml or run `klo setup demo init`',
|
||||
),
|
||||
);
|
||||
checks.push(check('pass', 'storage', 'Storage', `${project.config.storage.state}/${project.config.storage.search}`));
|
||||
checks.push(check('pass', 'llm-provider', 'LLM provider', project.config.llm.provider.backend));
|
||||
checks.push(await runSemanticSearchEmbeddingCheck(project.config.ingest.embeddings, projectDir, deps));
|
||||
const runHistoricSqlDoctorChecks =
|
||||
deps.runHistoricSqlDoctorChecks ?? (await import('./historic-sql-doctor.js')).runPostgresHistoricSqlDoctorChecks;
|
||||
checks.push(...(await runHistoricSqlDoctorChecks(project, deps)));
|
||||
} catch (error) {
|
||||
checks.push(
|
||||
check(
|
||||
'fail',
|
||||
'project-config',
|
||||
'Project config',
|
||||
failureMessage(error),
|
||||
`Run: klo init ${projectDir} --name <project-name>`,
|
||||
),
|
||||
);
|
||||
}
|
||||
return checks;
|
||||
}
|
||||
|
||||
async function runDemoProjectChecks(projectDir: string, deps: KloDoctorDeps = {}): Promise<DoctorCheck[]> {
|
||||
const env = deps.env ?? process.env;
|
||||
const { DEMO_CONNECTION_ID, DEMO_REPLAY_FILE } = await import('./demo-assets.js');
|
||||
const { loadKloProject } = await import('@klo/context/project');
|
||||
const checks: DoctorCheck[] = [];
|
||||
const requiredPaths = [
|
||||
['demo-config', 'Demo config', 'klo.yaml'],
|
||||
['demo-database', 'Demo dataset', 'demo.db'],
|
||||
['demo-state', 'Demo state database', 'state.sqlite'],
|
||||
['demo-replay', 'Demo replay', join('replays', DEMO_REPLAY_FILE)],
|
||||
['demo-raw-sources', 'Demo raw sources directory', 'raw-sources'],
|
||||
['demo-semantic-layer', 'Demo semantic-layer directory', 'semantic-layer'],
|
||||
['demo-knowledge', 'Demo knowledge directory', 'knowledge'],
|
||||
] as const;
|
||||
|
||||
for (const [id, label, relativePath] of requiredPaths) {
|
||||
const absolutePath = join(projectDir, relativePath);
|
||||
checks.push(
|
||||
(await defaultPathExists(absolutePath))
|
||||
? check('pass', id, label, relativePath)
|
||||
: check(
|
||||
'fail',
|
||||
id,
|
||||
label,
|
||||
`Missing ${relativePath}`,
|
||||
`Run: klo setup demo init --project-dir ${projectDir} --force --no-input`,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
const project = await loadKloProject({ projectDir });
|
||||
const connection = project.config.connections[DEMO_CONNECTION_ID];
|
||||
checks.push(
|
||||
connection?.driver === 'sqlite'
|
||||
? check('pass', 'demo-connection', 'Demo connection', `${DEMO_CONNECTION_ID} uses sqlite`)
|
||||
: check(
|
||||
'fail',
|
||||
'demo-connection',
|
||||
'Demo connection',
|
||||
`${DEMO_CONNECTION_ID} is missing or is not sqlite`,
|
||||
`Run: klo setup demo init --project-dir ${projectDir} --force --no-input`,
|
||||
),
|
||||
);
|
||||
const provider = project.config.llm.provider.backend;
|
||||
checks.push(
|
||||
provider === 'anthropic' || provider === 'vertex' || provider === 'gateway'
|
||||
? check('pass', 'demo-llm-provider', 'Demo LLM provider', provider)
|
||||
: check(
|
||||
'fail',
|
||||
'demo-llm-provider',
|
||||
'Demo LLM provider',
|
||||
provider,
|
||||
`Run: klo setup demo init --project-dir ${projectDir} --force --no-input`,
|
||||
),
|
||||
);
|
||||
if (provider === 'anthropic' && !env.ANTHROPIC_API_KEY) {
|
||||
checks.push(
|
||||
check(
|
||||
'warn',
|
||||
'anthropic-credentials',
|
||||
'Anthropic credentials',
|
||||
'ANTHROPIC_API_KEY is not set',
|
||||
'Export ANTHROPIC_API_KEY to run `klo setup demo --mode full --no-input`',
|
||||
),
|
||||
);
|
||||
} else {
|
||||
checks.push(check('pass', 'anthropic-credentials', 'Anthropic credentials', 'Configured for current provider'));
|
||||
}
|
||||
checks.push(await runSemanticSearchEmbeddingCheck(project.config.ingest.embeddings, projectDir, deps));
|
||||
const runHistoricSqlDoctorChecks =
|
||||
deps.runHistoricSqlDoctorChecks ?? (await import('./historic-sql-doctor.js')).runPostgresHistoricSqlDoctorChecks;
|
||||
checks.push(...(await runHistoricSqlDoctorChecks(project, deps)));
|
||||
} catch (error) {
|
||||
checks.push(
|
||||
check(
|
||||
'fail',
|
||||
'demo-config-parse',
|
||||
'Demo config parse',
|
||||
failureMessage(error),
|
||||
`Run: klo setup demo init --project-dir ${projectDir} --force --no-input`,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
return checks;
|
||||
}
|
||||
|
||||
export function formatDoctorReport(report: DoctorReport): string {
|
||||
const lines = [report.title];
|
||||
for (const item of report.checks) {
|
||||
lines.push(`${item.status.toUpperCase()} ${item.label}: ${item.detail}`);
|
||||
if (item.fix) {
|
||||
lines.push(` Fix: ${item.fix}`);
|
||||
}
|
||||
}
|
||||
lines.push('');
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
function hasFailures(report: DoctorReport): boolean {
|
||||
return report.checks.some((item) => item.status === 'fail');
|
||||
}
|
||||
|
||||
function writeReport(report: DoctorReport, outputMode: KloDoctorOutputMode, io: KloDoctorIo): void {
|
||||
if (outputMode === 'json') {
|
||||
io.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
|
||||
return;
|
||||
}
|
||||
io.stdout.write(formatDoctorReport(report));
|
||||
}
|
||||
|
||||
export async function runKloDoctor(
|
||||
args: KloDoctorArgs,
|
||||
io: KloDoctorIo = process,
|
||||
deps: KloDoctorDeps = {},
|
||||
): Promise<number> {
|
||||
try {
|
||||
const runSetupChecks = deps.runSetupChecks ?? (() => runSetupDoctorChecks());
|
||||
const setupChecks = await runSetupChecks();
|
||||
const report: DoctorReport =
|
||||
args.command === 'setup'
|
||||
? { title: 'KLO setup doctor', checks: setupChecks }
|
||||
: args.command === 'demo'
|
||||
? {
|
||||
title: 'KLO demo doctor',
|
||||
checks: [...setupChecks, ...(await runDemoProjectChecks(args.projectDir, deps))],
|
||||
}
|
||||
: {
|
||||
title: 'KLO project doctor',
|
||||
checks: [...setupChecks, ...(await runProjectChecks(args.projectDir, deps))],
|
||||
};
|
||||
|
||||
writeReport(report, args.outputMode, io);
|
||||
return hasFailures(report) ? 1 : 0;
|
||||
} catch (error) {
|
||||
io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
252
packages/cli/src/example-smoke.test.ts
Normal file
252
packages/cli/src/example-smoke.test.ts
Normal file
|
|
@ -0,0 +1,252 @@
|
|||
import { execFile } from 'node:child_process';
|
||||
import { cp, mkdtemp, rm } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join, resolve } from 'node:path';
|
||||
import { promisify } from 'node:util';
|
||||
import { Client } from '@modelcontextprotocol/sdk/client/index.js';
|
||||
import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js';
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
const CLI_BIN = resolve(process.cwd(), 'dist/bin.js');
|
||||
const EXAMPLE_DIR = resolve(process.cwd(), '../../examples/local-warehouse');
|
||||
|
||||
interface CliResult {
|
||||
code: number;
|
||||
stdout: string;
|
||||
stderr: string;
|
||||
}
|
||||
|
||||
interface ExecFailure extends Error {
|
||||
code?: number;
|
||||
stdout?: string;
|
||||
stderr?: string;
|
||||
}
|
||||
|
||||
function isExecFailure(error: unknown): error is ExecFailure {
|
||||
return error instanceof Error && ('stdout' in error || 'stderr' in error || 'code' in error);
|
||||
}
|
||||
|
||||
async function runBuiltCli(args: string[]): Promise<CliResult> {
|
||||
try {
|
||||
const result = await execFileAsync(process.execPath, [CLI_BIN, ...args], {
|
||||
encoding: 'utf8',
|
||||
timeout: 20_000,
|
||||
});
|
||||
return {
|
||||
code: 0,
|
||||
stdout: result.stdout,
|
||||
stderr: result.stderr,
|
||||
};
|
||||
} catch (error) {
|
||||
if (!isExecFailure(error)) {
|
||||
throw error;
|
||||
}
|
||||
return {
|
||||
code: typeof error.code === 'number' ? error.code : 1,
|
||||
stdout: error.stdout ?? '',
|
||||
stderr: error.stderr ?? error.message,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function structuredContent<T extends object>(result: unknown): T {
|
||||
const content = (result as { structuredContent?: unknown }).structuredContent;
|
||||
expect(content).toBeDefined();
|
||||
return content as T;
|
||||
}
|
||||
|
||||
function parseJsonOutput<T>(stdout: string): T {
|
||||
return JSON.parse(stdout) as T;
|
||||
}
|
||||
|
||||
async function copyExampleProject(tempDir: string): Promise<string> {
|
||||
const projectDir = join(tempDir, 'local-warehouse');
|
||||
await cp(EXAMPLE_DIR, projectDir, { recursive: true });
|
||||
return projectDir;
|
||||
}
|
||||
|
||||
describe('standalone local warehouse example', () => {
|
||||
let tempDir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'klo-example-smoke-'));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(tempDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('runs local CLI commands against the copied example project', async () => {
|
||||
const projectDir = await copyExampleProject(tempDir);
|
||||
const sourceDir = join(projectDir, 'source');
|
||||
|
||||
const knowledgeList = await runBuiltCli(['agent', 'wiki', 'search', 'revenue', '--json', '--project-dir', projectDir]);
|
||||
expect(knowledgeList).toMatchObject({ code: 0, stderr: '' });
|
||||
expect(parseJsonOutput<{ results: Array<{ key: string; summary: string }> }>(knowledgeList.stdout).results).toContainEqual(
|
||||
expect.objectContaining({ key: 'revenue', summary: 'Paid order value after refunds' }),
|
||||
);
|
||||
|
||||
const knowledgeRead = await runBuiltCli(['agent', 'wiki', 'read', 'revenue', '--json', '--project-dir', projectDir]);
|
||||
expect(knowledgeRead).toMatchObject({ code: 0, stderr: '' });
|
||||
expect(parseJsonOutput<{ content: string }>(knowledgeRead.stdout).content).toContain(
|
||||
'Revenue is paid order amount after refund adjustments.',
|
||||
);
|
||||
|
||||
const slList = await runBuiltCli(['agent', 'sl', 'list', '--json', '--project-dir', projectDir, '--connection-id', 'warehouse']);
|
||||
expect(slList).toMatchObject({ code: 0, stderr: '' });
|
||||
expect(parseJsonOutput<{ sources: Array<{ connectionId: string; name: string; columnCount: number }> }>(slList.stdout).sources).toContainEqual(
|
||||
expect.objectContaining({ connectionId: 'warehouse', name: 'orders', columnCount: 3 }),
|
||||
);
|
||||
|
||||
const slRead = await runBuiltCli([
|
||||
'agent',
|
||||
'sl',
|
||||
'read',
|
||||
'orders',
|
||||
'--json',
|
||||
'--connection-id',
|
||||
'warehouse',
|
||||
'--project-dir',
|
||||
projectDir,
|
||||
]);
|
||||
expect(slRead).toMatchObject({ code: 0, stderr: '' });
|
||||
expect(parseJsonOutput<{ yaml: string }>(slRead.stdout).yaml).toContain('name: orders');
|
||||
|
||||
const ingest = await runBuiltCli([
|
||||
'dev',
|
||||
'ingest',
|
||||
'run',
|
||||
'--project-dir',
|
||||
projectDir,
|
||||
'--connection-id',
|
||||
'warehouse',
|
||||
'--adapter',
|
||||
'fake',
|
||||
'--source-dir',
|
||||
sourceDir,
|
||||
]);
|
||||
expect(ingest).toMatchObject({ code: 1, stdout: '' });
|
||||
expect(ingest.stderr).toContain(
|
||||
'klo dev ingest run requires llm.provider.backend: anthropic, vertex, or gateway, or an injected agentRunner',
|
||||
);
|
||||
}, 30_000);
|
||||
|
||||
it('serves local wiki and semantic-layer MCP tools against the copied example project', async () => {
|
||||
const projectDir = await copyExampleProject(tempDir);
|
||||
|
||||
const client = new Client({ name: 'klo-example-client', version: '0.0.0' });
|
||||
const transport = new StdioClientTransport({
|
||||
command: process.execPath,
|
||||
args: [CLI_BIN, 'serve', '--mcp', 'stdio', '--project-dir', projectDir, '--user-id', 'example-user'],
|
||||
stderr: 'pipe',
|
||||
});
|
||||
|
||||
try {
|
||||
await client.connect(transport);
|
||||
|
||||
const knowledgeSearch = structuredContent<{
|
||||
results: Array<{ key: string; summary: string; score: number }>;
|
||||
totalFound: number;
|
||||
}>(
|
||||
await client.callTool({
|
||||
name: 'knowledge_search',
|
||||
arguments: { query: 'refund', limit: 5 },
|
||||
}),
|
||||
);
|
||||
expect(knowledgeSearch.totalFound).toBe(1);
|
||||
expect(knowledgeSearch.results[0]).toMatchObject({
|
||||
key: 'revenue',
|
||||
summary: 'Paid order value after refunds',
|
||||
});
|
||||
|
||||
const knowledgeRead = structuredContent<{ key: string; summary: string; content: string; scope: string }>(
|
||||
await client.callTool({ name: 'knowledge_read', arguments: { key: 'revenue' } }),
|
||||
);
|
||||
expect(knowledgeRead).toMatchObject({
|
||||
key: 'revenue',
|
||||
summary: 'Paid order value after refunds',
|
||||
scope: 'GLOBAL',
|
||||
});
|
||||
expect(knowledgeRead.content).toContain('Revenue is paid order amount after refund adjustments.');
|
||||
|
||||
const knowledgeWrite = structuredContent<{ success: boolean; key: string; action: string }>(
|
||||
await client.callTool({
|
||||
name: 'knowledge_write',
|
||||
arguments: {
|
||||
key: 'gross_margin',
|
||||
summary: 'Revenue after direct costs',
|
||||
content: 'Gross margin subtracts direct order costs from revenue.',
|
||||
tags: ['finance'],
|
||||
sl_refs: ['warehouse.orders'],
|
||||
},
|
||||
}),
|
||||
);
|
||||
expect(knowledgeWrite).toEqual({ success: true, key: 'gross_margin', action: 'created' });
|
||||
|
||||
const slList = structuredContent<{
|
||||
sources: Array<{
|
||||
connectionId: string;
|
||||
name: string;
|
||||
description?: string;
|
||||
columnCount: number;
|
||||
measureCount: number;
|
||||
joinCount: number;
|
||||
}>;
|
||||
totalSources: number;
|
||||
}>(await client.callTool({ name: 'sl_list_sources', arguments: { connectionId: 'warehouse' } }));
|
||||
expect(slList.totalSources).toBe(1);
|
||||
expect(slList.sources[0]).toMatchObject({
|
||||
connectionId: 'warehouse',
|
||||
name: 'orders',
|
||||
description: 'Orders placed through the storefront.',
|
||||
columnCount: 3,
|
||||
measureCount: 2,
|
||||
joinCount: 0,
|
||||
});
|
||||
|
||||
const slRead = structuredContent<{ sourceName: string; yaml: string }>(
|
||||
await client.callTool({
|
||||
name: 'sl_read_source',
|
||||
arguments: { connectionId: 'warehouse', sourceName: 'orders' },
|
||||
}),
|
||||
);
|
||||
expect(slRead.sourceName).toBe('orders');
|
||||
expect(slRead.yaml).toContain('name: orders');
|
||||
expect(slRead.yaml).toContain('total_revenue');
|
||||
|
||||
const slWrite = structuredContent<{ success: boolean; sourceName: string }>(
|
||||
await client.callTool({
|
||||
name: 'sl_write_source',
|
||||
arguments: {
|
||||
connectionId: 'warehouse',
|
||||
sourceName: 'customers',
|
||||
source: {
|
||||
name: 'customers',
|
||||
table: 'public.customers',
|
||||
grain: ['id'],
|
||||
columns: [{ name: 'id', type: 'number' }],
|
||||
joins: [],
|
||||
measures: [],
|
||||
},
|
||||
},
|
||||
}),
|
||||
);
|
||||
expect(slWrite).toMatchObject({ success: true, sourceName: 'customers' });
|
||||
|
||||
const slValidate = structuredContent<{ success: boolean; errors: string[]; warnings: string[] }>(
|
||||
await client.callTool({
|
||||
name: 'sl_validate',
|
||||
arguments: { connectionId: 'warehouse', names: ['orders', 'customers'] },
|
||||
}),
|
||||
);
|
||||
expect(slValidate.success).toBe(true);
|
||||
expect(slValidate.errors).toEqual([]);
|
||||
expect(slValidate.warnings).toContain(
|
||||
'Local stdio validation checks YAML shape only; Python semantic validation is not configured.',
|
||||
);
|
||||
} finally {
|
||||
await client.close();
|
||||
}
|
||||
}, 30_000);
|
||||
});
|
||||
173
packages/cli/src/historic-sql-doctor.test.ts
Normal file
173
packages/cli/src/historic-sql-doctor.test.ts
Normal file
|
|
@ -0,0 +1,173 @@
|
|||
import { buildDefaultKloProjectConfig, type KloProjectConnectionConfig } from '@klo/context/project';
|
||||
import { HistoricSqlExtensionMissingError } from '@klo/context/ingest';
|
||||
import { describe, expect, it, vi } from 'vitest';
|
||||
import {
|
||||
runPostgresHistoricSqlDoctorChecks,
|
||||
type HistoricSqlDoctorProject,
|
||||
type PostgresHistoricSqlDoctorProbe,
|
||||
} from './historic-sql-doctor.js';
|
||||
|
||||
function projectWithConnections(connections: Record<string, KloProjectConnectionConfig>): HistoricSqlDoctorProject {
|
||||
return {
|
||||
projectDir: '/tmp/klo-project',
|
||||
config: {
|
||||
...buildDefaultKloProjectConfig('warehouse'),
|
||||
connections,
|
||||
ingest: {
|
||||
...buildDefaultKloProjectConfig('warehouse').ingest,
|
||||
adapters: ['live-database', 'historic-sql'],
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe('runPostgresHistoricSqlDoctorChecks', () => {
|
||||
it('passes when no Postgres historic-SQL connections are enabled', async () => {
|
||||
const checks = await runPostgresHistoricSqlDoctorChecks(
|
||||
projectWithConnections({
|
||||
warehouse: { driver: 'sqlite', path: './warehouse.db', readonly: true },
|
||||
}),
|
||||
{
|
||||
postgresHistoricSqlProbe: vi.fn<PostgresHistoricSqlDoctorProbe>(),
|
||||
},
|
||||
);
|
||||
|
||||
expect(checks).toEqual([
|
||||
{
|
||||
id: 'historic-sql-postgres',
|
||||
label: 'Postgres Historic SQL',
|
||||
status: 'pass',
|
||||
detail: 'No enabled Postgres historic-SQL connections',
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it('passes when the PGSS probe succeeds without warnings', async () => {
|
||||
const probe = vi.fn<PostgresHistoricSqlDoctorProbe>(async () => ({
|
||||
pgServerVersion: 'PostgreSQL 16.4',
|
||||
warnings: [],
|
||||
}));
|
||||
|
||||
const checks = await runPostgresHistoricSqlDoctorChecks(
|
||||
projectWithConnections({
|
||||
warehouse: {
|
||||
driver: 'postgres',
|
||||
url: 'env:WAREHOUSE_DATABASE_URL',
|
||||
readonly: true,
|
||||
historicSql: { enabled: true, dialect: 'postgres' },
|
||||
},
|
||||
}),
|
||||
{ postgresHistoricSqlProbe: probe },
|
||||
);
|
||||
|
||||
expect(probe).toHaveBeenCalledWith({
|
||||
projectDir: '/tmp/klo-project',
|
||||
connectionId: 'warehouse',
|
||||
connection: {
|
||||
driver: 'postgres',
|
||||
url: 'env:WAREHOUSE_DATABASE_URL',
|
||||
readonly: true,
|
||||
historicSql: { enabled: true, dialect: 'postgres' },
|
||||
},
|
||||
env: process.env,
|
||||
});
|
||||
expect(checks).toEqual([
|
||||
{
|
||||
id: 'historic-sql-postgres-warehouse',
|
||||
label: 'Postgres Historic SQL (warehouse)',
|
||||
status: 'pass',
|
||||
detail: 'pg_stat_statements ready (PostgreSQL 16.4)',
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it('warns when the PGSS probe succeeds with operational warnings', async () => {
|
||||
const checks = await runPostgresHistoricSqlDoctorChecks(
|
||||
projectWithConnections({
|
||||
warehouse: {
|
||||
driver: 'postgres',
|
||||
url: 'env:WAREHOUSE_DATABASE_URL',
|
||||
readonly: true,
|
||||
historicSql: { enabled: true, dialect: 'postgres' },
|
||||
},
|
||||
}),
|
||||
{
|
||||
postgresHistoricSqlProbe: async () => ({
|
||||
pgServerVersion: 'PostgreSQL 16.4',
|
||||
warnings: [
|
||||
'pg_stat_statements.max is 1000; set it to at least 5000 to reduce query-template eviction churn',
|
||||
],
|
||||
}),
|
||||
},
|
||||
);
|
||||
|
||||
expect(checks).toEqual([
|
||||
{
|
||||
id: 'historic-sql-postgres-warehouse',
|
||||
label: 'Postgres Historic SQL (warehouse)',
|
||||
status: 'warn',
|
||||
detail:
|
||||
'pg_stat_statements ready (PostgreSQL 16.4) with warnings: pg_stat_statements.max is 1000; set it to at least 5000 to reduce query-template eviction churn',
|
||||
fix: 'Update the Postgres parameter group or config, then rerun `klo dev doctor --project-dir /tmp/klo-project`',
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it('fails when a connection has postgres historic SQL but is not a Postgres driver', async () => {
|
||||
const checks = await runPostgresHistoricSqlDoctorChecks(
|
||||
projectWithConnections({
|
||||
warehouse: {
|
||||
driver: 'mysql',
|
||||
url: 'env:WAREHOUSE_DATABASE_URL',
|
||||
readonly: true,
|
||||
historicSql: { enabled: true, dialect: 'postgres' },
|
||||
},
|
||||
}),
|
||||
{
|
||||
postgresHistoricSqlProbe: vi.fn<PostgresHistoricSqlDoctorProbe>(),
|
||||
},
|
||||
);
|
||||
|
||||
expect(checks).toEqual([
|
||||
{
|
||||
id: 'historic-sql-postgres-warehouse',
|
||||
label: 'Postgres Historic SQL (warehouse)',
|
||||
status: 'fail',
|
||||
detail: 'connections.warehouse.historicSql.dialect is postgres but driver is mysql',
|
||||
fix: 'Set connections.warehouse.driver to postgres or disable historicSql for this connection',
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it('maps PGSS capability errors to actionable failures', async () => {
|
||||
const checks = await runPostgresHistoricSqlDoctorChecks(
|
||||
projectWithConnections({
|
||||
warehouse: {
|
||||
driver: 'postgres',
|
||||
url: 'env:WAREHOUSE_DATABASE_URL',
|
||||
readonly: true,
|
||||
historicSql: { enabled: true, dialect: 'postgres' },
|
||||
},
|
||||
}),
|
||||
{
|
||||
postgresHistoricSqlProbe: async () => {
|
||||
throw new HistoricSqlExtensionMissingError({
|
||||
dialect: 'postgres',
|
||||
message: 'pg_stat_statements extension is not installed in the connection database.',
|
||||
remediation: 'Run CREATE EXTENSION pg_stat_statements; against the connection database.',
|
||||
});
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
expect(checks).toEqual([
|
||||
{
|
||||
id: 'historic-sql-postgres-warehouse',
|
||||
label: 'Postgres Historic SQL (warehouse)',
|
||||
status: 'fail',
|
||||
detail: 'pg_stat_statements extension is not installed in the connection database.',
|
||||
fix: 'Run CREATE EXTENSION pg_stat_statements; against the connection database.',
|
||||
},
|
||||
]);
|
||||
});
|
||||
});
|
||||
160
packages/cli/src/historic-sql-doctor.ts
Normal file
160
packages/cli/src/historic-sql-doctor.ts
Normal file
|
|
@ -0,0 +1,160 @@
|
|||
import type { KloProjectConfig, KloProjectConnectionConfig } from '@klo/context/project';
|
||||
import type { DoctorCheck } from './doctor.js';
|
||||
|
||||
export interface HistoricSqlDoctorProject {
|
||||
projectDir: string;
|
||||
config: Pick<KloProjectConfig, 'connections' | 'ingest'>;
|
||||
}
|
||||
|
||||
export interface PostgresHistoricSqlDoctorProbeInput {
|
||||
projectDir: string;
|
||||
connectionId: string;
|
||||
connection: KloProjectConnectionConfig;
|
||||
env: NodeJS.ProcessEnv;
|
||||
}
|
||||
|
||||
export interface PostgresHistoricSqlDoctorProbeResult {
|
||||
pgServerVersion: string;
|
||||
warnings: string[];
|
||||
}
|
||||
|
||||
export type PostgresHistoricSqlDoctorProbe = (
|
||||
input: PostgresHistoricSqlDoctorProbeInput,
|
||||
) => Promise<PostgresHistoricSqlDoctorProbeResult>;
|
||||
|
||||
export interface HistoricSqlDoctorDeps {
|
||||
env?: NodeJS.ProcessEnv;
|
||||
postgresHistoricSqlProbe?: PostgresHistoricSqlDoctorProbe;
|
||||
}
|
||||
|
||||
function check(status: DoctorCheck['status'], id: string, label: string, detail: string, fix?: string): DoctorCheck {
|
||||
return fix ? { id, label, status, detail, fix } : { id, label, status, detail };
|
||||
}
|
||||
|
||||
function historicSqlRecord(connection: KloProjectConnectionConfig): Record<string, unknown> | null {
|
||||
const historicSql = connection.historicSql;
|
||||
return historicSql && typeof historicSql === 'object' && !Array.isArray(historicSql)
|
||||
? (historicSql as Record<string, unknown>)
|
||||
: null;
|
||||
}
|
||||
|
||||
function isEnabledPostgresHistoricSql(connection: KloProjectConnectionConfig): boolean {
|
||||
const historicSql = historicSqlRecord(connection);
|
||||
return historicSql?.enabled === true && historicSql.dialect === 'postgres';
|
||||
}
|
||||
|
||||
function isPostgresDriver(connection: KloProjectConnectionConfig): boolean {
|
||||
const driver = String(connection.driver ?? '').toLowerCase();
|
||||
return driver === 'postgres' || driver === 'postgresql';
|
||||
}
|
||||
|
||||
function checkId(connectionId: string): string {
|
||||
return `historic-sql-postgres-${connectionId.replace(/[^a-z0-9_-]+/gi, '-')}`;
|
||||
}
|
||||
|
||||
function capabilityFailureFix(error: unknown, connectionId: string, projectDir: string): string {
|
||||
if (error instanceof Error && error.name === 'HistoricSqlExtensionMissingError' && 'remediation' in error) {
|
||||
return String(error.remediation);
|
||||
}
|
||||
if (error instanceof Error && error.name === 'HistoricSqlGrantsMissingError' && 'remediation' in error) {
|
||||
return String(error.remediation);
|
||||
}
|
||||
if (error instanceof Error && error.name === 'HistoricSqlVersionUnsupportedError') {
|
||||
return 'Use PostgreSQL 14 or newer, or disable historicSql for this connection';
|
||||
}
|
||||
return `Fix connections.${connectionId} Postgres settings, then rerun \`klo dev doctor --project-dir ${projectDir}\``;
|
||||
}
|
||||
|
||||
function failureDetail(error: unknown): string {
|
||||
if (error instanceof Error && error.message.trim().length > 0) {
|
||||
return error.message.trim().split('\n')[0] ?? error.message.trim();
|
||||
}
|
||||
return String(error);
|
||||
}
|
||||
|
||||
async function defaultPostgresHistoricSqlProbe(
|
||||
input: PostgresHistoricSqlDoctorProbeInput,
|
||||
): Promise<PostgresHistoricSqlDoctorProbeResult> {
|
||||
const [{ PostgresPgssQueryHistoryReader }, { KloPostgresHistoricSqlQueryClient, isKloPostgresConnectionConfig }] =
|
||||
await Promise.all([import('@klo/context/ingest'), import('@klo/connector-postgres')]);
|
||||
|
||||
if (!isKloPostgresConnectionConfig(input.connection)) {
|
||||
throw new Error(`Native PostgreSQL connector cannot run driver "${input.connection.driver ?? 'unknown'}"`);
|
||||
}
|
||||
|
||||
const client = new KloPostgresHistoricSqlQueryClient({
|
||||
connectionId: input.connectionId,
|
||||
connection: input.connection,
|
||||
env: input.env,
|
||||
});
|
||||
try {
|
||||
return await new PostgresPgssQueryHistoryReader().probe(client);
|
||||
} finally {
|
||||
await client.cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
export async function runPostgresHistoricSqlDoctorChecks(
|
||||
project: HistoricSqlDoctorProject,
|
||||
deps: HistoricSqlDoctorDeps = {},
|
||||
): Promise<DoctorCheck[]> {
|
||||
const targets = Object.entries(project.config.connections)
|
||||
.filter(([, connection]) => isEnabledPostgresHistoricSql(connection))
|
||||
.sort(([left], [right]) => left.localeCompare(right));
|
||||
|
||||
if (targets.length === 0) {
|
||||
return [
|
||||
check('pass', 'historic-sql-postgres', 'Postgres Historic SQL', 'No enabled Postgres historic-SQL connections'),
|
||||
];
|
||||
}
|
||||
|
||||
const probe = deps.postgresHistoricSqlProbe ?? defaultPostgresHistoricSqlProbe;
|
||||
const env = deps.env ?? process.env;
|
||||
const checks: DoctorCheck[] = [];
|
||||
for (const [connectionId, connection] of targets) {
|
||||
const label = `Postgres Historic SQL (${connectionId})`;
|
||||
if (!isPostgresDriver(connection)) {
|
||||
checks.push(
|
||||
check(
|
||||
'fail',
|
||||
checkId(connectionId),
|
||||
label,
|
||||
`connections.${connectionId}.historicSql.dialect is postgres but driver is ${String(connection.driver)}`,
|
||||
`Set connections.${connectionId}.driver to postgres or disable historicSql for this connection`,
|
||||
),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await probe({ projectDir: project.projectDir, connectionId, connection, env });
|
||||
if (result.warnings.length > 0) {
|
||||
checks.push(
|
||||
check(
|
||||
'warn',
|
||||
checkId(connectionId),
|
||||
label,
|
||||
`pg_stat_statements ready (${result.pgServerVersion}) with warnings: ${result.warnings.join('; ')}`,
|
||||
`Update the Postgres parameter group or config, then rerun \`klo dev doctor --project-dir ${project.projectDir}\``,
|
||||
),
|
||||
);
|
||||
} else {
|
||||
checks.push(
|
||||
check('pass', checkId(connectionId), label, `pg_stat_statements ready (${result.pgServerVersion})`),
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
checks.push(
|
||||
check(
|
||||
'fail',
|
||||
checkId(connectionId),
|
||||
label,
|
||||
failureDetail(error),
|
||||
capabilityFailureFix(error, connectionId, project.projectDir),
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return checks;
|
||||
}
|
||||
1977
packages/cli/src/index.test.ts
Normal file
1977
packages/cli/src/index.test.ts
Normal file
File diff suppressed because it is too large
Load diff
53
packages/cli/src/index.ts
Normal file
53
packages/cli/src/index.ts
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
import { profileMark } from './startup-profile.js';
|
||||
|
||||
export {
|
||||
getKloCliPackageInfo,
|
||||
runInitForCommander,
|
||||
runKloCli,
|
||||
type KloCliDeps,
|
||||
type KloCliIo,
|
||||
type KloCliPackageInfo,
|
||||
} from './cli-runtime.js';
|
||||
export { runKloAgent, type KloAgentArgs } from './agent.js';
|
||||
export {
|
||||
KLO_AGENT_MAX_ROWS_CAP,
|
||||
createKloAgentRuntime,
|
||||
parseAgentMaxRows,
|
||||
readAgentJsonFile,
|
||||
writeAgentJson,
|
||||
writeAgentJsonError,
|
||||
type KloAgentRuntime,
|
||||
type KloAgentRuntimeDeps,
|
||||
} from './agent-runtime.js';
|
||||
export { runKloSetup, type KloSetupArgs, type KloSetupStatus } from './setup.js';
|
||||
export type {
|
||||
KloSetupDatabaseDriver,
|
||||
KloSetupDatabasesArgs,
|
||||
KloSetupDatabasesDeps,
|
||||
KloSetupDatabasesResult,
|
||||
} from './setup-databases.js';
|
||||
export { runKloSetupDatabasesStep } from './setup-databases.js';
|
||||
export type {
|
||||
KloSetupEmbeddingBackend,
|
||||
KloSetupEmbeddingsArgs,
|
||||
KloSetupEmbeddingsDeps,
|
||||
KloSetupEmbeddingsResult,
|
||||
} from './setup-embeddings.js';
|
||||
export { runKloSetupEmbeddingsStep } from './setup-embeddings.js';
|
||||
export type {
|
||||
KloSetupSourcesArgs,
|
||||
KloSetupSourcesDeps,
|
||||
KloSetupSourcesPromptAdapter,
|
||||
KloSetupSourcesResult,
|
||||
KloSetupSourceType,
|
||||
} from './setup-sources.js';
|
||||
export { runKloSetupSourcesStep } from './setup-sources.js';
|
||||
export type { KloMemoryFlowTuiIo, MemoryFlowTuiLiveSession } from './memory-flow-tui.js';
|
||||
export {
|
||||
renderMemoryFlowTui,
|
||||
sanitizeMemoryFlowTuiError,
|
||||
startLiveMemoryFlowTui,
|
||||
} from './memory-flow-tui.js';
|
||||
export { rendererUnavailableVizFallback, resolveVizFallback, warnVizFallbackOnce } from './viz-fallback.js';
|
||||
|
||||
profileMark('module:index');
|
||||
78
packages/cli/src/ingest-report-file.test.ts
Normal file
78
packages/cli/src/ingest-report-file.test.ts
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
import { mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
||||
import { readIngestReportSnapshotFile } from './ingest-report-file.js';
|
||||
|
||||
function reportSnapshot() {
|
||||
return {
|
||||
id: 'report-1',
|
||||
runId: 'run-1',
|
||||
jobId: 'job-1',
|
||||
connectionId: 'warehouse',
|
||||
sourceKey: 'metabase',
|
||||
createdAt: '2026-04-30T12:00:00.000Z',
|
||||
body: {
|
||||
syncId: 'sync-1',
|
||||
diffSummary: { added: 1, modified: 0, deleted: 0, unchanged: 0 },
|
||||
commitSha: null,
|
||||
workUnits: [],
|
||||
failedWorkUnits: [],
|
||||
reconciliationSkipped: true,
|
||||
conflictsResolved: [],
|
||||
evictionsApplied: [],
|
||||
unmappedFallbacks: [],
|
||||
evictionInputs: [],
|
||||
unresolvedCards: [],
|
||||
supersededBy: null,
|
||||
overrideOf: null,
|
||||
provenanceRows: [],
|
||||
toolTranscripts: [],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe('readIngestReportSnapshotFile', () => {
|
||||
let tempDir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'klo-report-file-'));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(tempDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('reads and parses an ingest report JSON file', async () => {
|
||||
const reportPath = join(tempDir, 'report.json');
|
||||
await writeFile(reportPath, `${JSON.stringify(reportSnapshot(), null, 2)}\n`, 'utf-8');
|
||||
|
||||
const report = await readIngestReportSnapshotFile(reportPath);
|
||||
|
||||
expect(report).toMatchObject({
|
||||
id: 'report-1',
|
||||
runId: 'run-1',
|
||||
jobId: 'job-1',
|
||||
connectionId: 'warehouse',
|
||||
sourceKey: 'metabase',
|
||||
});
|
||||
});
|
||||
|
||||
it('reports invalid JSON with the file path', async () => {
|
||||
const reportPath = join(tempDir, 'invalid.json');
|
||||
await writeFile(reportPath, '{not json', 'utf-8');
|
||||
|
||||
await expect(readIngestReportSnapshotFile(reportPath)).rejects.toThrow(
|
||||
`Invalid JSON in ingest report file ${reportPath}`,
|
||||
);
|
||||
});
|
||||
|
||||
it('reports schema failures with the file path', async () => {
|
||||
const reportPath = join(tempDir, 'wrong-shape.json');
|
||||
await writeFile(reportPath, JSON.stringify({ id: 'report-1' }), 'utf-8');
|
||||
|
||||
await expect(readIngestReportSnapshotFile(reportPath)).rejects.toThrow(
|
||||
`Invalid ingest report file ${reportPath}`,
|
||||
);
|
||||
});
|
||||
});
|
||||
20
packages/cli/src/ingest-report-file.ts
Normal file
20
packages/cli/src/ingest-report-file.ts
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
import { readFile } from 'node:fs/promises';
|
||||
import { parseIngestReportSnapshot, type IngestReportSnapshot } from '@klo/context/ingest';
|
||||
|
||||
export async function readIngestReportSnapshotFile(reportFile: string): Promise<IngestReportSnapshot> {
|
||||
const raw = await readFile(reportFile, 'utf-8');
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(raw);
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
throw new Error(`Invalid JSON in ingest report file ${reportFile}: ${message}`);
|
||||
}
|
||||
|
||||
try {
|
||||
return parseIngestReportSnapshot(parsed);
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
throw new Error(`Invalid ingest report file ${reportFile}: ${message}`);
|
||||
}
|
||||
}
|
||||
2275
packages/cli/src/ingest.test.ts
Normal file
2275
packages/cli/src/ingest.test.ts
Normal file
File diff suppressed because it is too large
Load diff
425
packages/cli/src/ingest.ts
Normal file
425
packages/cli/src/ingest.ts
Normal file
|
|
@ -0,0 +1,425 @@
|
|||
import {
|
||||
buildMemoryFlowViewModel,
|
||||
createMemoryFlowLiveBuffer,
|
||||
formatMemoryFlowFinalSummary,
|
||||
getLatestLocalIngestStatus,
|
||||
getLocalIngestStatus,
|
||||
type IngestReportSnapshot,
|
||||
ingestReportToMemoryFlowReplay,
|
||||
type LocalMetabaseFanoutResult,
|
||||
type LocalMetabaseFanoutProgress,
|
||||
type MemoryFlowReplayInput,
|
||||
type RunLocalIngestOptions,
|
||||
renderMemoryFlowReplay,
|
||||
runLocalIngest,
|
||||
runLocalMetabaseIngest,
|
||||
} from '@klo/context/ingest';
|
||||
import { loadKloProject } from '@klo/context/project';
|
||||
import { readIngestReportSnapshotFile } from './ingest-report-file.js';
|
||||
import { createKloCliLocalIngestAdapters } from './local-adapters.js';
|
||||
import { type KloMemoryFlowStdin, renderMemoryFlowInteractively } from './memory-flow-interactive.js';
|
||||
import {
|
||||
type KloMemoryFlowTuiIo,
|
||||
type MemoryFlowTuiLiveSession,
|
||||
renderMemoryFlowTui,
|
||||
startLiveMemoryFlowTui,
|
||||
} from './memory-flow-tui.js';
|
||||
import { resolveVizFallback, warnVizFallbackOnce } from './viz-fallback.js';
|
||||
import { profileMark } from './startup-profile.js';
|
||||
|
||||
profileMark('module:ingest');
|
||||
|
||||
export type KloIngestOutputMode = 'plain' | 'json' | 'viz';
|
||||
type KloIngestInputMode = 'auto' | 'disabled';
|
||||
|
||||
export type KloIngestArgs =
|
||||
| {
|
||||
command: 'run';
|
||||
projectDir: string;
|
||||
connectionId: string;
|
||||
adapter: string;
|
||||
sourceDir?: string;
|
||||
databaseIntrospectionUrl?: string;
|
||||
debugLlmRequestFile?: string;
|
||||
outputMode: KloIngestOutputMode;
|
||||
inputMode?: KloIngestInputMode;
|
||||
}
|
||||
| {
|
||||
command: 'status' | 'replay' | 'watch';
|
||||
projectDir: string;
|
||||
runId?: string;
|
||||
reportFile?: string;
|
||||
outputMode: KloIngestOutputMode;
|
||||
inputMode?: KloIngestInputMode;
|
||||
};
|
||||
|
||||
interface KloIngestIo {
|
||||
stdin?: KloMemoryFlowStdin;
|
||||
stdout: { isTTY?: boolean; columns?: number; write(chunk: string): void };
|
||||
stderr: { write(chunk: string): void };
|
||||
}
|
||||
|
||||
interface KloIngestDeps {
|
||||
jobIdFactory?: () => string;
|
||||
now?: () => Date;
|
||||
createAdapters?: typeof createKloCliLocalIngestAdapters;
|
||||
runLocalIngest?: typeof runLocalIngest;
|
||||
runLocalMetabaseIngest?: typeof runLocalMetabaseIngest;
|
||||
readReportFile?: typeof readIngestReportSnapshotFile;
|
||||
renderStoredMemoryFlow?: typeof renderMemoryFlowTui;
|
||||
startLiveMemoryFlow?: typeof startLiveMemoryFlowTui;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
localIngestOptions?: Pick<
|
||||
RunLocalIngestOptions,
|
||||
| 'agentRunner'
|
||||
| 'llmProvider'
|
||||
| 'memoryModel'
|
||||
| 'semanticLayerCompute'
|
||||
| 'queryExecutor'
|
||||
| 'logger'
|
||||
| 'pullConfigOptions'
|
||||
>;
|
||||
}
|
||||
|
||||
function reportStatus(report: IngestReportSnapshot): 'done' | 'error' {
|
||||
return report.body.failedWorkUnits.length > 0 ? 'error' : 'done';
|
||||
}
|
||||
|
||||
function reportActionCounts(report: IngestReportSnapshot): { wikiCount: number; slCount: number } {
|
||||
const actions = report.body.workUnits.flatMap((workUnit) => workUnit.actions);
|
||||
return {
|
||||
wikiCount: actions.filter((action) => action.target === 'wiki').length,
|
||||
slCount: actions.filter((action) => action.target === 'sl').length,
|
||||
};
|
||||
}
|
||||
|
||||
function writeReportStatus(report: IngestReportSnapshot, io: KloIngestIo): void {
|
||||
const counts = reportActionCounts(report);
|
||||
io.stdout.write(`Report: ${report.id}\n`);
|
||||
io.stdout.write(`Run: ${report.runId}\n`);
|
||||
io.stdout.write(`Job: ${report.jobId}\n`);
|
||||
io.stdout.write(`Status: ${reportStatus(report)}\n`);
|
||||
io.stdout.write(`Adapter: ${report.sourceKey}\n`);
|
||||
io.stdout.write(`Connection: ${report.connectionId}\n`);
|
||||
io.stdout.write(`Sync: ${report.body.syncId}\n`);
|
||||
io.stdout.write(
|
||||
`Diff: +${report.body.diffSummary.added}/~${report.body.diffSummary.modified}/-${report.body.diffSummary.deleted}/=${report.body.diffSummary.unchanged}\n`,
|
||||
);
|
||||
io.stdout.write(`Work units: ${report.body.workUnits.length}\n`);
|
||||
io.stdout.write(`Saved memory: ${counts.wikiCount} wiki, ${counts.slCount} SL\n`);
|
||||
io.stdout.write(`Provenance rows: ${report.body.provenanceRows.length}\n`);
|
||||
}
|
||||
|
||||
function writeMetabaseFanoutStatus(result: LocalMetabaseFanoutResult, io: KloIngestIo): void {
|
||||
io.stdout.write(`Metabase fan-out: ${result.status}\n`);
|
||||
io.stdout.write(`Source: ${result.metabaseConnectionId}\n`);
|
||||
io.stdout.write(`Children: ${result.children.length}\n`);
|
||||
if (result.totals) {
|
||||
io.stdout.write(`Work units: ${result.totals.workUnits}\n`);
|
||||
io.stdout.write(`Failed work units: ${result.totals.failedWorkUnits}\n`);
|
||||
}
|
||||
for (const child of result.children) {
|
||||
const status = reportStatus(child.report);
|
||||
io.stdout.write(
|
||||
`- target=${child.targetConnectionId} database=${child.metabaseDatabaseId} status=${status} job=${child.jobId}\n`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function pluralize(count: number, singular: string, plural = `${singular}s`): string {
|
||||
return `${count} ${count === 1 ? singular : plural}`;
|
||||
}
|
||||
|
||||
function createMetabaseFanoutProgress(
|
||||
connectionId: string,
|
||||
io: KloIngestIo,
|
||||
): LocalMetabaseFanoutProgress {
|
||||
io.stdout.write(`Metabase ingest: ${connectionId}\n`);
|
||||
io.stdout.write('Checking mappings and scheduled-pull targets...\n');
|
||||
return {
|
||||
onMetabaseFanoutPlanned(event) {
|
||||
io.stdout.write(`Targets: ${pluralize(event.children.length, 'mapped database')}\n`);
|
||||
for (const child of event.children) {
|
||||
io.stdout.write(`- database=${child.metabaseDatabaseId} target=${child.targetConnectionId} status=queued\n`);
|
||||
}
|
||||
},
|
||||
onMetabaseChildStarted(event) {
|
||||
io.stdout.write(
|
||||
`- database=${event.metabaseDatabaseId} target=${event.targetConnectionId} status=running job=${event.jobId}\n`,
|
||||
);
|
||||
},
|
||||
onMetabaseChildCompleted(event) {
|
||||
io.stdout.write(
|
||||
`- database=${event.metabaseDatabaseId} target=${event.targetConnectionId} status=${event.status} job=${event.jobId}\n`,
|
||||
);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function writeReportJson(report: IngestReportSnapshot, io: KloIngestIo): void {
|
||||
io.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
|
||||
}
|
||||
|
||||
function assertReportMatchesReplayId(report: IngestReportSnapshot, requestedId: string, reportFile: string): void {
|
||||
const validIds = [report.id, report.runId, report.jobId];
|
||||
if (!validIds.includes(requestedId)) {
|
||||
throw new Error(
|
||||
`Report file ${reportFile} does not match ingest replay id "${requestedId}"; expected one of ${validIds.join(
|
||||
', ',
|
||||
)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function readStoredIngestReport(
|
||||
project: Awaited<ReturnType<typeof loadKloProject>>,
|
||||
runId: string | undefined,
|
||||
): Promise<IngestReportSnapshot | null> {
|
||||
return runId ? await getLocalIngestStatus(project, runId) : await getLatestLocalIngestStatus(project);
|
||||
}
|
||||
|
||||
function isInteractiveTerminal(io: KloIngestIo): boolean {
|
||||
return io.stdout.isTTY === true;
|
||||
}
|
||||
|
||||
function terminalWidth(io: KloIngestIo): number | undefined {
|
||||
return io.stdout.columns ?? process.stdout.columns;
|
||||
}
|
||||
|
||||
function isTuiCapableIo(io: KloIngestIo): io is KloIngestIo & KloMemoryFlowTuiIo {
|
||||
return (
|
||||
io.stdin?.isTTY === true &&
|
||||
io.stdout.isTTY === true &&
|
||||
typeof io.stdin.on === 'function' &&
|
||||
typeof io.stdin.setRawMode === 'function' &&
|
||||
typeof io.stdout.write === 'function'
|
||||
);
|
||||
}
|
||||
|
||||
interface EffectiveIngestOutputModeOptions {
|
||||
requireInput?: boolean;
|
||||
}
|
||||
|
||||
function effectiveIngestOutputMode(
|
||||
outputMode: KloIngestOutputMode,
|
||||
io: KloIngestIo,
|
||||
env: NodeJS.ProcessEnv,
|
||||
options: EffectiveIngestOutputModeOptions = {},
|
||||
): KloIngestOutputMode {
|
||||
if (outputMode !== 'viz') {
|
||||
return outputMode;
|
||||
}
|
||||
|
||||
const fallback = resolveVizFallback(io, env, { requireInput: options.requireInput ?? false });
|
||||
if (!fallback.shouldDegrade) {
|
||||
return outputMode;
|
||||
}
|
||||
|
||||
warnVizFallbackOnce(io, fallback);
|
||||
return 'plain';
|
||||
}
|
||||
|
||||
function writeMemoryFlowInput(input: MemoryFlowReplayInput, io: KloIngestIo, options: { clear?: boolean } = {}): void {
|
||||
if (options.clear) {
|
||||
io.stdout.write('\u001b[2J\u001b[H');
|
||||
}
|
||||
const view = buildMemoryFlowViewModel(input);
|
||||
io.stdout.write(renderMemoryFlowReplay(view, { terminalWidth: terminalWidth(io) }));
|
||||
}
|
||||
|
||||
function initialRunMemoryFlowInput(
|
||||
args: Extract<KloIngestArgs, { command: 'run' }>,
|
||||
runId: string,
|
||||
): MemoryFlowReplayInput {
|
||||
return {
|
||||
runId,
|
||||
connectionId: args.connectionId,
|
||||
adapter: args.adapter,
|
||||
status: 'running',
|
||||
sourceDir: args.sourceDir ?? null,
|
||||
syncId: 'pending',
|
||||
errors: [],
|
||||
events: [],
|
||||
plannedWorkUnits: [],
|
||||
details: { actions: [], provenance: [], transcripts: [] },
|
||||
};
|
||||
}
|
||||
|
||||
async function writeReportRecord(
|
||||
report: IngestReportSnapshot,
|
||||
outputMode: KloIngestOutputMode,
|
||||
io: KloIngestIo,
|
||||
options: {
|
||||
interactive?: boolean;
|
||||
renderStoredMemoryFlow?: typeof renderMemoryFlowTui;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
} = {},
|
||||
): Promise<void> {
|
||||
if (outputMode === 'json') {
|
||||
writeReportJson(report, io);
|
||||
return;
|
||||
}
|
||||
|
||||
const resolvedOutputMode = effectiveIngestOutputMode(outputMode, io, options.env ?? process.env, {
|
||||
requireInput: options.interactive === true,
|
||||
});
|
||||
|
||||
if (resolvedOutputMode === 'viz') {
|
||||
const input = ingestReportToMemoryFlowReplay(report, { provenanceRowCount: report.body.provenanceRows.length });
|
||||
if (options.interactive === true) {
|
||||
if (io.stdin?.isTTY === true) {
|
||||
const renderStoredMemoryFlow = options.renderStoredMemoryFlow ?? renderMemoryFlowTui;
|
||||
if (isTuiCapableIo(io) && (await renderStoredMemoryFlow(input, io))) {
|
||||
return;
|
||||
}
|
||||
|
||||
await renderMemoryFlowInteractively(input, io);
|
||||
return;
|
||||
}
|
||||
|
||||
writeMemoryFlowInput(input, io);
|
||||
return;
|
||||
}
|
||||
|
||||
writeMemoryFlowInput(input, io);
|
||||
return;
|
||||
}
|
||||
|
||||
writeReportStatus(report, io);
|
||||
}
|
||||
|
||||
export async function runKloIngest(
|
||||
args: KloIngestArgs,
|
||||
io: KloIngestIo = process,
|
||||
deps: KloIngestDeps = {},
|
||||
): Promise<number> {
|
||||
try {
|
||||
const project = await loadKloProject({ projectDir: args.projectDir });
|
||||
const env = deps.env ?? process.env;
|
||||
if (args.command === 'run') {
|
||||
const createAdapters = deps.createAdapters ?? createKloCliLocalIngestAdapters;
|
||||
const executeLocalIngest = deps.runLocalIngest ?? runLocalIngest;
|
||||
const localIngestOptions = deps.localIngestOptions ?? {};
|
||||
const adapterOptions = {
|
||||
...(localIngestOptions.pullConfigOptions ?? {}),
|
||||
...(args.databaseIntrospectionUrl ? { databaseIntrospectionUrl: args.databaseIntrospectionUrl } : {}),
|
||||
...(args.adapter === 'historic-sql' ? { historicSqlConnectionId: args.connectionId } : {}),
|
||||
};
|
||||
if (args.adapter === 'metabase' && args.sourceDir) {
|
||||
throw new Error('source-dir uploads are not supported for the Metabase fan-out adapter');
|
||||
}
|
||||
if (args.adapter === 'metabase') {
|
||||
const executeMetabaseFanout = deps.runLocalMetabaseIngest ?? runLocalMetabaseIngest;
|
||||
const progress =
|
||||
args.outputMode === 'json' ? undefined : createMetabaseFanoutProgress(args.connectionId, io);
|
||||
const result = await executeMetabaseFanout({
|
||||
project,
|
||||
adapters: createAdapters(project, adapterOptions),
|
||||
metabaseConnectionId: args.connectionId,
|
||||
...localIngestOptions,
|
||||
trigger: 'manual_resync',
|
||||
jobIdFactory: deps.jobIdFactory,
|
||||
...(progress ? { progress } : {}),
|
||||
});
|
||||
if (args.outputMode === 'json') {
|
||||
io.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
|
||||
} else {
|
||||
writeMetabaseFanoutStatus(result, io);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
const jobId = deps.jobIdFactory?.();
|
||||
let liveTui: MemoryFlowTuiLiveSession | null = null;
|
||||
const runOutputMode = effectiveIngestOutputMode(args.outputMode, io, env, {
|
||||
requireInput: (args.inputMode ?? 'auto') === 'auto',
|
||||
});
|
||||
const shouldUseLiveViz =
|
||||
runOutputMode === 'viz' && (args.inputMode ?? 'auto') === 'auto' && isInteractiveTerminal(io);
|
||||
const initialMemoryFlow = shouldUseLiveViz ? initialRunMemoryFlowInput(args, jobId ?? 'pending') : undefined;
|
||||
let latestMemoryFlowSnapshot: MemoryFlowReplayInput | null = initialMemoryFlow ?? null;
|
||||
|
||||
if (initialMemoryFlow && isTuiCapableIo(io)) {
|
||||
const startLiveMemoryFlow = deps.startLiveMemoryFlow ?? startLiveMemoryFlowTui;
|
||||
liveTui = await startLiveMemoryFlow(initialMemoryFlow, io);
|
||||
}
|
||||
|
||||
const memoryFlow = initialMemoryFlow
|
||||
? createMemoryFlowLiveBuffer(initialMemoryFlow, {
|
||||
onChange: (snapshot) => {
|
||||
latestMemoryFlowSnapshot = snapshot;
|
||||
if (liveTui && !liveTui.isClosed()) {
|
||||
liveTui.update(snapshot);
|
||||
return;
|
||||
}
|
||||
if (!liveTui) {
|
||||
writeMemoryFlowInput(snapshot, io, { clear: true });
|
||||
}
|
||||
},
|
||||
})
|
||||
: undefined;
|
||||
|
||||
try {
|
||||
const result = await executeLocalIngest({
|
||||
project,
|
||||
adapters: createAdapters(project, adapterOptions),
|
||||
adapter: args.adapter,
|
||||
connectionId: args.connectionId,
|
||||
sourceDir: args.sourceDir,
|
||||
trigger: 'manual_resync',
|
||||
jobId,
|
||||
...localIngestOptions,
|
||||
...(args.debugLlmRequestFile ? { llmDebugRequestFile: args.debugLlmRequestFile } : {}),
|
||||
...(memoryFlow ? { memoryFlow } : {}),
|
||||
});
|
||||
if (memoryFlow) {
|
||||
latestMemoryFlowSnapshot = memoryFlow.snapshot();
|
||||
liveTui?.close();
|
||||
liveTui = null;
|
||||
io.stdout.write(formatMemoryFlowFinalSummary(latestMemoryFlowSnapshot));
|
||||
return 0;
|
||||
}
|
||||
await writeReportRecord(result.report, runOutputMode, io, {
|
||||
interactive: (args.inputMode ?? 'auto') === 'auto',
|
||||
renderStoredMemoryFlow: deps.renderStoredMemoryFlow,
|
||||
env,
|
||||
});
|
||||
return 0;
|
||||
} finally {
|
||||
liveTui?.close();
|
||||
}
|
||||
}
|
||||
|
||||
if (args.reportFile) {
|
||||
const readReportFile = deps.readReportFile ?? readIngestReportSnapshotFile;
|
||||
const report = await readReportFile(args.reportFile);
|
||||
if (args.runId) {
|
||||
assertReportMatchesReplayId(report, args.runId, args.reportFile);
|
||||
}
|
||||
await writeReportRecord(report, args.outputMode, io, {
|
||||
interactive: (args.inputMode ?? 'auto') === 'auto',
|
||||
renderStoredMemoryFlow: deps.renderStoredMemoryFlow,
|
||||
env,
|
||||
});
|
||||
return 0;
|
||||
}
|
||||
|
||||
const report = await readStoredIngestReport(project, args.runId);
|
||||
if (!report) {
|
||||
throw new Error(
|
||||
args.runId
|
||||
? `Local ingest run or report "${args.runId}" was not found`
|
||||
: 'No local ingest reports were found. Run `klo ingest --all` first.',
|
||||
);
|
||||
}
|
||||
await writeReportRecord(report, args.outputMode, io, {
|
||||
interactive: (args.inputMode ?? 'auto') === 'auto',
|
||||
renderStoredMemoryFlow: deps.renderStoredMemoryFlow,
|
||||
env,
|
||||
});
|
||||
return 0;
|
||||
} catch (error) {
|
||||
io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
60
packages/cli/src/io/mode.test.ts
Normal file
60
packages/cli/src/io/mode.test.ts
Normal file
|
|
@ -0,0 +1,60 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import type { KloCliIo } from '../cli-runtime.js';
|
||||
import { resolveOutputMode } from './mode.js';
|
||||
|
||||
function ioWith(isTTY: boolean | undefined): KloCliIo {
|
||||
return {
|
||||
stdout: { isTTY, write: () => {} },
|
||||
stderr: { write: () => {} },
|
||||
};
|
||||
}
|
||||
|
||||
describe('resolveOutputMode', () => {
|
||||
it('uses explicit value when provided', () => {
|
||||
expect(resolveOutputMode({ explicit: 'pretty', io: ioWith(false), env: {} })).toBe('pretty');
|
||||
expect(resolveOutputMode({ explicit: 'plain', io: ioWith(true), env: {} })).toBe('plain');
|
||||
expect(resolveOutputMode({ explicit: 'json', io: ioWith(true), env: {} })).toBe('json');
|
||||
});
|
||||
|
||||
it('json:true takes precedence over explicit value', () => {
|
||||
expect(resolveOutputMode({ explicit: 'pretty', json: true, io: ioWith(true), env: {} })).toBe('json');
|
||||
});
|
||||
|
||||
it('throws on unknown explicit value', () => {
|
||||
expect(() => resolveOutputMode({ explicit: 'fancy', io: ioWith(true), env: {} })).toThrow(/Invalid --output/);
|
||||
});
|
||||
|
||||
it('honors KLO_OUTPUT env var when no explicit value', () => {
|
||||
expect(resolveOutputMode({ io: ioWith(true), env: { KLO_OUTPUT: 'plain' } })).toBe('plain');
|
||||
expect(resolveOutputMode({ io: ioWith(false), env: { KLO_OUTPUT: 'pretty' } })).toBe('pretty');
|
||||
expect(resolveOutputMode({ io: ioWith(false), env: { KLO_OUTPUT: 'json' } })).toBe('json');
|
||||
});
|
||||
|
||||
it('throws on unknown KLO_OUTPUT', () => {
|
||||
expect(() => resolveOutputMode({ io: ioWith(true), env: { KLO_OUTPUT: 'fancy' } })).toThrow(/Invalid KLO_OUTPUT/);
|
||||
});
|
||||
|
||||
it('returns plain when CI is set to a truthy value', () => {
|
||||
expect(resolveOutputMode({ io: ioWith(true), env: { CI: 'true' } })).toBe('plain');
|
||||
expect(resolveOutputMode({ io: ioWith(true), env: { CI: '1' } })).toBe('plain');
|
||||
});
|
||||
|
||||
it('ignores CI when set to a falsy value', () => {
|
||||
expect(resolveOutputMode({ io: ioWith(true), env: { CI: '' } })).toBe('pretty');
|
||||
expect(resolveOutputMode({ io: ioWith(true), env: { CI: '0' } })).toBe('pretty');
|
||||
expect(resolveOutputMode({ io: ioWith(true), env: { CI: 'false' } })).toBe('pretty');
|
||||
});
|
||||
|
||||
it('returns pretty when stdout is a TTY and CI is not set', () => {
|
||||
expect(resolveOutputMode({ io: ioWith(true), env: {} })).toBe('pretty');
|
||||
});
|
||||
|
||||
it('returns plain when stdout is not a TTY', () => {
|
||||
expect(resolveOutputMode({ io: ioWith(false), env: {} })).toBe('plain');
|
||||
expect(resolveOutputMode({ io: ioWith(undefined), env: {} })).toBe('plain');
|
||||
});
|
||||
|
||||
it('explicit value beats KLO_OUTPUT env var', () => {
|
||||
expect(resolveOutputMode({ explicit: 'json', io: ioWith(true), env: { KLO_OUTPUT: 'plain' } })).toBe('json');
|
||||
});
|
||||
});
|
||||
40
packages/cli/src/io/mode.ts
Normal file
40
packages/cli/src/io/mode.ts
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
import type { KloCliIo } from '../cli-runtime.js';
|
||||
|
||||
export type KloOutputMode = 'pretty' | 'plain' | 'json';
|
||||
|
||||
const MODES: ReadonlySet<string> = new Set(['pretty', 'plain', 'json']);
|
||||
|
||||
export interface ResolveOutputModeArgs {
|
||||
explicit?: string;
|
||||
json?: boolean;
|
||||
io: KloCliIo;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
}
|
||||
|
||||
export function resolveOutputMode(args: ResolveOutputModeArgs): KloOutputMode {
|
||||
if (args.json === true) {
|
||||
return 'json';
|
||||
}
|
||||
if (args.explicit !== undefined) {
|
||||
if (!MODES.has(args.explicit)) {
|
||||
throw new Error(`Invalid --output value: ${args.explicit}. Expected one of pretty, plain, json.`);
|
||||
}
|
||||
return args.explicit as KloOutputMode;
|
||||
}
|
||||
const env = args.env ?? process.env;
|
||||
const envMode = env.KLO_OUTPUT;
|
||||
if (envMode !== undefined && envMode !== '') {
|
||||
if (!MODES.has(envMode)) {
|
||||
throw new Error(`Invalid KLO_OUTPUT value: ${envMode}. Expected one of pretty, plain, json.`);
|
||||
}
|
||||
return envMode as KloOutputMode;
|
||||
}
|
||||
const ci = env.CI;
|
||||
if (ci !== undefined && ci !== '' && ci !== '0' && ci !== 'false') {
|
||||
return 'plain';
|
||||
}
|
||||
if (args.io.stdout.isTTY === true) {
|
||||
return 'pretty';
|
||||
}
|
||||
return 'plain';
|
||||
}
|
||||
171
packages/cli/src/io/print-list.test.ts
Normal file
171
packages/cli/src/io/print-list.test.ts
Normal file
|
|
@ -0,0 +1,171 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import type { KloCliIo } from '../cli-runtime.js';
|
||||
import { printList, type PrintListColumn } from './print-list.js';
|
||||
import { SYMBOLS } from './symbols.js';
|
||||
|
||||
function recorder(): { io: KloCliIo; out: () => string; err: () => string } {
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
return {
|
||||
io: {
|
||||
stdout: { write: (chunk: string) => { stdout += chunk; } },
|
||||
stderr: { write: (chunk: string) => { stderr += chunk; } },
|
||||
},
|
||||
out: () => stdout,
|
||||
err: () => stderr,
|
||||
};
|
||||
}
|
||||
|
||||
interface SlRow {
|
||||
connectionId: string;
|
||||
name: string;
|
||||
columnCount: number;
|
||||
measureCount: number;
|
||||
joinCount: number;
|
||||
description?: string;
|
||||
}
|
||||
|
||||
const SL_COLUMNS: ReadonlyArray<PrintListColumn<SlRow>> = [
|
||||
{ key: 'connectionId', label: 'CONNECTION', plain: '' },
|
||||
{ key: 'name', label: 'NAME', plain: '' },
|
||||
{ key: 'columnCount', label: 'COLS', plain: 'columns=', dim: true },
|
||||
{ key: 'measureCount', label: 'MEASURES', plain: 'measures=', dim: true },
|
||||
{ key: 'joinCount', label: 'JOINS', plain: 'joins=', dim: true },
|
||||
{ key: 'description', label: 'DESCRIPTION', plain: false, optional: true, dim: true },
|
||||
];
|
||||
|
||||
const ORDERS: SlRow = { connectionId: 'warehouse', name: 'orders', columnCount: 5, measureCount: 3, joinCount: 1 };
|
||||
const USERS: SlRow = { connectionId: 'warehouse', name: 'users', columnCount: 8, measureCount: 2, joinCount: 2, description: 'User profile + auth' };
|
||||
|
||||
describe('printList — plain mode', () => {
|
||||
it('emits one tab-separated row per item, skipping plain:false columns', () => {
|
||||
const r = recorder();
|
||||
printList<SlRow>({
|
||||
rows: [ORDERS, USERS],
|
||||
columns: SL_COLUMNS,
|
||||
mode: 'plain',
|
||||
command: 'sl list',
|
||||
emptyMessage: 'No sources',
|
||||
io: r.io,
|
||||
});
|
||||
expect(r.out()).toBe(
|
||||
'warehouse\torders\tcolumns=5\tmeasures=3\tjoins=1\n' +
|
||||
'warehouse\tusers\tcolumns=8\tmeasures=2\tjoins=2\n',
|
||||
);
|
||||
});
|
||||
|
||||
it('emits nothing on empty list (preserves current sl list zero-row behavior)', () => {
|
||||
const r = recorder();
|
||||
printList<SlRow>({
|
||||
rows: [],
|
||||
columns: SL_COLUMNS,
|
||||
mode: 'plain',
|
||||
command: 'sl list',
|
||||
emptyMessage: 'No sources',
|
||||
io: r.io,
|
||||
});
|
||||
expect(r.out()).toBe('');
|
||||
});
|
||||
});
|
||||
|
||||
describe('printList — json mode', () => {
|
||||
it('emits the envelope with kind=list, data.items, and meta.command', () => {
|
||||
const r = recorder();
|
||||
printList<SlRow>({
|
||||
rows: [ORDERS, USERS],
|
||||
columns: SL_COLUMNS,
|
||||
mode: 'json',
|
||||
command: 'sl list',
|
||||
emptyMessage: 'No sources',
|
||||
io: r.io,
|
||||
});
|
||||
const written = r.out();
|
||||
expect(written.endsWith('\n')).toBe(true);
|
||||
const parsed = JSON.parse(written);
|
||||
expect(parsed).toEqual({
|
||||
kind: 'list',
|
||||
data: { items: [ORDERS, USERS] },
|
||||
meta: { command: 'sl list' },
|
||||
});
|
||||
});
|
||||
|
||||
it('emits an empty items array when no rows', () => {
|
||||
const r = recorder();
|
||||
printList<SlRow>({
|
||||
rows: [],
|
||||
columns: SL_COLUMNS,
|
||||
mode: 'json',
|
||||
command: 'sl list',
|
||||
emptyMessage: 'No sources',
|
||||
io: r.io,
|
||||
});
|
||||
expect(JSON.parse(r.out())).toEqual({
|
||||
kind: 'list',
|
||||
data: { items: [] },
|
||||
meta: { command: 'sl list' },
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
function stripAnsi(s: string): string {
|
||||
// Matches ESC [ ... m sequences emitted by node:util.styleText.
|
||||
return s.replace(/\[[0-9;]*m/g, '');
|
||||
}
|
||||
|
||||
describe('printList — pretty mode', () => {
|
||||
it('renders a Clack-style header, grouped rows, and footer', () => {
|
||||
const r = recorder();
|
||||
printList<SlRow>({
|
||||
rows: [ORDERS, USERS],
|
||||
columns: SL_COLUMNS,
|
||||
groupBy: 'connectionId',
|
||||
mode: 'pretty',
|
||||
command: 'sl list',
|
||||
emptyMessage: 'No sources',
|
||||
io: r.io,
|
||||
});
|
||||
const out = stripAnsi(r.out());
|
||||
expect(out).toContain(`${SYMBOLS.barStart} sl list`);
|
||||
expect(out).toContain(`${SYMBOLS.group} warehouse`);
|
||||
expect(out).toContain('(2 sources)');
|
||||
expect(out).toMatch(new RegExp(`${escapeRegExp(SYMBOLS.item)} orders\\s+5 cols ${escapeRegExp(SYMBOLS.middot)} 3 measures ${escapeRegExp(SYMBOLS.middot)} 1 join\\b`));
|
||||
expect(out).toMatch(new RegExp(`${escapeRegExp(SYMBOLS.item)} users\\s+8 cols ${escapeRegExp(SYMBOLS.middot)} 2 measures ${escapeRegExp(SYMBOLS.middot)} 2 joins\\b`));
|
||||
expect(out).toContain(`${SYMBOLS.emDash} User profile + auth`);
|
||||
expect(out).toContain(`${SYMBOLS.barEnd} 2 sources`);
|
||||
});
|
||||
|
||||
it('renders an empty-state message when no rows', () => {
|
||||
const r = recorder();
|
||||
printList<SlRow>({
|
||||
rows: [],
|
||||
columns: SL_COLUMNS,
|
||||
groupBy: 'connectionId',
|
||||
mode: 'pretty',
|
||||
command: 'sl list',
|
||||
emptyMessage: 'No semantic-layer sources found in /tmp/proj',
|
||||
io: r.io,
|
||||
});
|
||||
const out = stripAnsi(r.out());
|
||||
expect(out).toContain(`${SYMBOLS.barStart} sl list`);
|
||||
expect(out).toContain(`${SYMBOLS.barEnd} No semantic-layer sources found in /tmp/proj`);
|
||||
});
|
||||
|
||||
it('singularizes the footer when there is one row', () => {
|
||||
const r = recorder();
|
||||
printList<SlRow>({
|
||||
rows: [ORDERS],
|
||||
columns: SL_COLUMNS,
|
||||
groupBy: 'connectionId',
|
||||
mode: 'pretty',
|
||||
command: 'sl list',
|
||||
emptyMessage: 'No sources',
|
||||
io: r.io,
|
||||
});
|
||||
const out = stripAnsi(r.out());
|
||||
expect(out).toContain(`${SYMBOLS.barEnd} 1 source`);
|
||||
});
|
||||
});
|
||||
|
||||
function escapeRegExp(s: string): string {
|
||||
return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
}
|
||||
164
packages/cli/src/io/print-list.ts
Normal file
164
packages/cli/src/io/print-list.ts
Normal file
|
|
@ -0,0 +1,164 @@
|
|||
import type { KloCliIo } from '../cli-runtime.js';
|
||||
import type { KloOutputMode } from './mode.js';
|
||||
import { bold, dim, SYMBOLS } from './symbols.js';
|
||||
|
||||
export interface PrintListColumn<Row> {
|
||||
key: keyof Row & string;
|
||||
label?: string;
|
||||
/**
|
||||
* Plain-mode rendering control.
|
||||
* - `string` (including `''`): emit `${plain}${value}` as a tab-separated cell.
|
||||
* - `false`: omit this column entirely in plain mode.
|
||||
* - `undefined`: same as `''`.
|
||||
*/
|
||||
plain?: string | false;
|
||||
/** Skip this column when the row's value is null / undefined / empty string. */
|
||||
optional?: boolean;
|
||||
/** Pretty-mode hint: render this column dim. */
|
||||
dim?: boolean;
|
||||
}
|
||||
|
||||
export interface PrintListArgs<Row> {
|
||||
rows: ReadonlyArray<Row>;
|
||||
columns: ReadonlyArray<PrintListColumn<Row>>;
|
||||
groupBy?: keyof Row & string;
|
||||
emptyMessage: string;
|
||||
command: string;
|
||||
mode: KloOutputMode;
|
||||
io: KloCliIo;
|
||||
}
|
||||
|
||||
export function printList<Row extends object>(args: PrintListArgs<Row>): void {
|
||||
switch (args.mode) {
|
||||
case 'json':
|
||||
printListJson(args);
|
||||
return;
|
||||
case 'plain':
|
||||
printListPlain(args);
|
||||
return;
|
||||
case 'pretty':
|
||||
printListPretty(args);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
function isEmpty(value: unknown): boolean {
|
||||
return value === undefined || value === null || value === '';
|
||||
}
|
||||
|
||||
function printListPlain<Row extends object>(args: PrintListArgs<Row>): void {
|
||||
for (const row of args.rows) {
|
||||
const cells: string[] = [];
|
||||
for (const col of args.columns) {
|
||||
if (col.plain === false) continue;
|
||||
const value = row[col.key];
|
||||
if (col.optional && isEmpty(value)) continue;
|
||||
const prefix = col.plain ?? '';
|
||||
cells.push(`${prefix}${value === undefined || value === null ? '' : String(value)}`);
|
||||
}
|
||||
args.io.stdout.write(`${cells.join('\t')}\n`);
|
||||
}
|
||||
}
|
||||
|
||||
function printListJson<Row extends object>(args: PrintListArgs<Row>): void {
|
||||
const envelope = {
|
||||
kind: 'list',
|
||||
data: { items: args.rows },
|
||||
meta: { command: args.command },
|
||||
};
|
||||
args.io.stdout.write(`${JSON.stringify(envelope, null, 2)}\n`);
|
||||
}
|
||||
|
||||
function pluralize(count: number, singular: string): string {
|
||||
return `${count} ${count === 1 ? singular : `${singular}s`}`;
|
||||
}
|
||||
|
||||
function metricCell(label: string, count: number): string {
|
||||
// "5 cols", "3 measures", "1 join" / "2 joins"
|
||||
// The label in PrintListColumn is uppercase; pretty mode lowercases it.
|
||||
const word = label.toLowerCase();
|
||||
return `${count} ${count === 1 ? singularize(word) : word}`;
|
||||
}
|
||||
|
||||
function singularize(word: string): string {
|
||||
if (word === 'joins') return 'join';
|
||||
if (word === 'measures') return 'measure';
|
||||
if (word === 'cols') return 'col';
|
||||
if (word.endsWith('s')) return word.slice(0, -1);
|
||||
return word;
|
||||
}
|
||||
|
||||
function groupRows<Row extends object>(
|
||||
rows: ReadonlyArray<Row>,
|
||||
key: keyof Row & string,
|
||||
): Map<string, Row[]> {
|
||||
const groups = new Map<string, Row[]>();
|
||||
for (const row of rows) {
|
||||
const value = String(row[key] ?? '');
|
||||
const bucket = groups.get(value);
|
||||
if (bucket) {
|
||||
bucket.push(row);
|
||||
} else {
|
||||
groups.set(value, [row]);
|
||||
}
|
||||
}
|
||||
return groups;
|
||||
}
|
||||
|
||||
function printListPretty<Row extends object>(args: PrintListArgs<Row>): void {
|
||||
const { io, command, rows, columns, groupBy, emptyMessage } = args;
|
||||
|
||||
io.stdout.write(`${SYMBOLS.barStart} ${command}\n`);
|
||||
io.stdout.write(`${SYMBOLS.bar}\n`);
|
||||
|
||||
if (rows.length === 0) {
|
||||
io.stdout.write(`${SYMBOLS.barEnd} ${emptyMessage}\n`);
|
||||
return;
|
||||
}
|
||||
|
||||
// Identify role of each column.
|
||||
// - First non-grouped, non-metric, non-optional column = "name" column (bolded)
|
||||
// - Columns with a `plain` prefix = metric columns (rendered as "N word")
|
||||
// - optional columns = trailing suffix (em-dash + value), only when value is present
|
||||
const nameCol = columns.find(
|
||||
(c) => c.key !== groupBy && !c.plain && !c.optional && c.plain !== false,
|
||||
);
|
||||
const metricCols = columns.filter((c) => typeof c.plain === 'string' && c.plain.length > 0);
|
||||
const optionalCols = columns.filter((c) => c.optional === true);
|
||||
|
||||
const buckets = groupBy ? groupRows(rows, groupBy) : new Map<string, Row[]>([['', [...rows]]]);
|
||||
|
||||
const nameWidth = nameCol
|
||||
? Math.max(...rows.map((r) => String(r[nameCol.key] ?? '').length))
|
||||
: 0;
|
||||
|
||||
for (const [groupValue, groupRowList] of buckets) {
|
||||
if (groupBy) {
|
||||
io.stdout.write(
|
||||
`${SYMBOLS.bar} ${SYMBOLS.group} ${bold(groupValue)} ${dim(`(${pluralize(groupRowList.length, 'source')})`)}\n`,
|
||||
);
|
||||
}
|
||||
for (const row of groupRowList) {
|
||||
const segments: string[] = [];
|
||||
if (nameCol) {
|
||||
segments.push(String(row[nameCol.key] ?? '').padEnd(nameWidth));
|
||||
}
|
||||
const metrics = metricCols
|
||||
.map((c) => metricCell(c.label ?? c.key, Number(row[c.key] ?? 0)))
|
||||
.join(` ${SYMBOLS.middot} `);
|
||||
if (metrics.length > 0) segments.push(dim(metrics));
|
||||
const optionalSuffix = optionalCols
|
||||
.map((c) => row[c.key])
|
||||
.filter((v) => !isEmpty(v))
|
||||
.map((v) => `${SYMBOLS.emDash} ${dim(String(v))}`)
|
||||
.join(' ');
|
||||
if (optionalSuffix.length > 0) segments.push(optionalSuffix);
|
||||
|
||||
const indent = groupBy ? ' ' : ' ';
|
||||
io.stdout.write(`${SYMBOLS.bar}${indent}${SYMBOLS.item} ${segments.join(' ')}\n`);
|
||||
}
|
||||
}
|
||||
|
||||
io.stdout.write(`${SYMBOLS.bar}\n`);
|
||||
io.stdout.write(`${SYMBOLS.barEnd} ${pluralize(rows.length, 'source')}\n`);
|
||||
}
|
||||
37
packages/cli/src/io/symbols.ts
Normal file
37
packages/cli/src/io/symbols.ts
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
import { styleText } from 'node:util';
|
||||
|
||||
function detectUnicodeSupport(env: NodeJS.ProcessEnv = process.env): boolean {
|
||||
if (process.platform !== 'win32') {
|
||||
return env.TERM !== 'linux';
|
||||
}
|
||||
return (
|
||||
Boolean(env.WT_SESSION) ||
|
||||
env.TERM_PROGRAM === 'vscode' ||
|
||||
env.TERM === 'xterm-256color' ||
|
||||
env.TERM === 'alacritty'
|
||||
);
|
||||
}
|
||||
|
||||
const unicode = detectUnicodeSupport();
|
||||
|
||||
export const SYMBOLS = {
|
||||
bar: unicode ? '│' : '|',
|
||||
barStart: unicode ? '◇' : 'o',
|
||||
barEnd: unicode ? '└' : '—',
|
||||
group: unicode ? '●' : '*',
|
||||
item: unicode ? '◆' : '*',
|
||||
middot: unicode ? '·' : '-',
|
||||
emDash: unicode ? '—' : '--',
|
||||
} as const;
|
||||
|
||||
export function dim(text: string): string {
|
||||
return styleText('dim', text);
|
||||
}
|
||||
|
||||
export function bold(text: string): string {
|
||||
return styleText('bold', text);
|
||||
}
|
||||
|
||||
export function gray(text: string): string {
|
||||
return styleText('gray', text);
|
||||
}
|
||||
95
packages/cli/src/knowledge.test.ts
Normal file
95
packages/cli/src/knowledge.test.ts
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
import { mkdtemp, rm } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { initKloProject } from '@klo/context/project';
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
||||
import { runKloKnowledge } from './knowledge.js';
|
||||
|
||||
function makeIo() {
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
return {
|
||||
io: {
|
||||
stdout: {
|
||||
write: (chunk: string) => {
|
||||
stdout += chunk;
|
||||
},
|
||||
},
|
||||
stderr: {
|
||||
write: (chunk: string) => {
|
||||
stderr += chunk;
|
||||
},
|
||||
},
|
||||
},
|
||||
stdout: () => stdout,
|
||||
stderr: () => stderr,
|
||||
};
|
||||
}
|
||||
|
||||
describe('runKloKnowledge', () => {
|
||||
let tempDir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'klo-cli-knowledge-'));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(tempDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('writes, reads, lists, and searches knowledge pages', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await initKloProject({ projectDir, projectName: 'warehouse' });
|
||||
|
||||
const writeIo = makeIo();
|
||||
await expect(
|
||||
runKloKnowledge(
|
||||
{
|
||||
command: 'write',
|
||||
projectDir,
|
||||
key: 'metrics/revenue',
|
||||
scope: 'GLOBAL',
|
||||
userId: 'local',
|
||||
summary: 'Revenue',
|
||||
content: 'Revenue is paid order value.',
|
||||
tags: ['finance'],
|
||||
refs: [],
|
||||
slRefs: ['orders'],
|
||||
},
|
||||
writeIo.io,
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
expect(writeIo.stdout()).toContain('Wrote knowledge/global/metrics/revenue.md');
|
||||
|
||||
const readIo = makeIo();
|
||||
await expect(
|
||||
runKloKnowledge({ command: 'read', projectDir, key: 'metrics/revenue', userId: 'local' }, readIo.io),
|
||||
).resolves.toBe(0);
|
||||
expect(readIo.stdout()).toContain('# metrics/revenue');
|
||||
expect(readIo.stdout()).toContain('Revenue is paid order value.');
|
||||
|
||||
const listIo = makeIo();
|
||||
await expect(runKloKnowledge({ command: 'list', projectDir, userId: 'local' }, listIo.io)).resolves.toBe(0);
|
||||
expect(listIo.stdout()).toContain('GLOBAL\tmetrics/revenue\tRevenue');
|
||||
|
||||
const searchIo = makeIo();
|
||||
await expect(
|
||||
runKloKnowledge({ command: 'search', projectDir, query: 'paid order', userId: 'local' }, searchIo.io),
|
||||
).resolves.toBe(0);
|
||||
expect(searchIo.stdout()).toContain('metrics/revenue');
|
||||
});
|
||||
|
||||
it('explains empty search results for a project without wiki pages', async () => {
|
||||
const projectDir = join(tempDir, 'empty-project');
|
||||
await initKloProject({ projectDir, projectName: 'warehouse' });
|
||||
|
||||
const searchIo = makeIo();
|
||||
await expect(
|
||||
runKloKnowledge({ command: 'search', projectDir, query: 'revenue', userId: 'local' }, searchIo.io),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(searchIo.stdout()).toBe('');
|
||||
expect(searchIo.stderr()).toContain('No local wiki pages found');
|
||||
expect(searchIo.stderr()).toContain('klo wiki write');
|
||||
});
|
||||
});
|
||||
90
packages/cli/src/knowledge.ts
Normal file
90
packages/cli/src/knowledge.ts
Normal file
|
|
@ -0,0 +1,90 @@
|
|||
import { loadKloProject } from '@klo/context/project';
|
||||
import {
|
||||
type LocalKnowledgeScope,
|
||||
listLocalKnowledgePages,
|
||||
readLocalKnowledgePage,
|
||||
searchLocalKnowledgePages,
|
||||
writeLocalKnowledgePage,
|
||||
} from '@klo/context/wiki';
|
||||
|
||||
export type KloKnowledgeArgs =
|
||||
| { command: 'list'; projectDir: string; userId: string }
|
||||
| { command: 'read'; projectDir: string; key: string; userId: string }
|
||||
| { command: 'search'; projectDir: string; query: string; userId: string }
|
||||
| {
|
||||
command: 'write';
|
||||
projectDir: string;
|
||||
key: string;
|
||||
scope: LocalKnowledgeScope;
|
||||
userId: string;
|
||||
summary: string;
|
||||
content: string;
|
||||
tags: string[];
|
||||
refs: string[];
|
||||
slRefs: string[];
|
||||
};
|
||||
|
||||
interface KloKnowledgeIo {
|
||||
stdout: { write(chunk: string): void };
|
||||
stderr: { write(chunk: string): void };
|
||||
}
|
||||
|
||||
export async function runKloKnowledge(args: KloKnowledgeArgs, io: KloKnowledgeIo = process): Promise<number> {
|
||||
try {
|
||||
const project = await loadKloProject({ projectDir: args.projectDir });
|
||||
if (args.command === 'list') {
|
||||
const pages = await listLocalKnowledgePages(project, { userId: args.userId });
|
||||
for (const page of pages) {
|
||||
io.stdout.write(`${page.scope}\t${page.key}\t${page.summary}\n`);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
if (args.command === 'read') {
|
||||
const page = await readLocalKnowledgePage(project, { key: args.key, userId: args.userId });
|
||||
if (!page) {
|
||||
throw new Error(`Knowledge page "${args.key}" was not found`);
|
||||
}
|
||||
io.stdout.write(`# ${page.key}\n\n`);
|
||||
io.stdout.write(`Scope: ${page.scope}\n`);
|
||||
io.stdout.write(`Summary: ${page.summary}\n\n`);
|
||||
io.stdout.write(`${page.content}\n`);
|
||||
return 0;
|
||||
}
|
||||
if (args.command === 'search') {
|
||||
const results = await searchLocalKnowledgePages(project, { query: args.query, userId: args.userId });
|
||||
if (results.length === 0) {
|
||||
const pages = await listLocalKnowledgePages(project, { userId: args.userId });
|
||||
if (pages.length === 0) {
|
||||
io.stderr.write(
|
||||
`No local wiki pages found in ${project.projectDir}. Create one with \`klo wiki write <key> --summary <summary> --content <content>\` or run ingest.\n`,
|
||||
);
|
||||
} else {
|
||||
io.stderr.write(
|
||||
`No local wiki pages matched "${args.query}". Run \`klo wiki list\` to inspect available pages.\n`,
|
||||
);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
for (const result of results) {
|
||||
io.stdout.write(`${result.score}\t${result.scope}\t${result.key}\t${result.summary}\n`);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
const write = await writeLocalKnowledgePage(project, {
|
||||
key: args.key,
|
||||
scope: args.scope,
|
||||
userId: args.userId,
|
||||
summary: args.summary,
|
||||
content: args.content,
|
||||
tags: args.tags,
|
||||
refs: args.refs,
|
||||
slRefs: args.slRefs,
|
||||
});
|
||||
io.stdout.write(`Wrote ${write.path}\n`);
|
||||
return 0;
|
||||
} catch (error) {
|
||||
io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
173
packages/cli/src/local-adapters.ts
Normal file
173
packages/cli/src/local-adapters.ts
Normal file
|
|
@ -0,0 +1,173 @@
|
|||
import { join } from 'node:path';
|
||||
import { createBigQueryLiveDatabaseIntrospection, isKloBigQueryConnectionConfig } from '@klo/connector-bigquery';
|
||||
import { createClickHouseLiveDatabaseIntrospection, isKloClickHouseConnectionConfig } from '@klo/connector-clickhouse';
|
||||
import { createMysqlLiveDatabaseIntrospection, isKloMysqlConnectionConfig } from '@klo/connector-mysql';
|
||||
import {
|
||||
createPostgresLiveDatabaseIntrospection,
|
||||
isKloPostgresConnectionConfig,
|
||||
type KloPostgresConnectionConfig,
|
||||
KloPostgresHistoricSqlQueryClient,
|
||||
} from '@klo/connector-postgres';
|
||||
import { createSqliteLiveDatabaseIntrospection, isKloSqliteConnectionConfig } from '@klo/connector-sqlite';
|
||||
import { createSqlServerLiveDatabaseIntrospection, isKloSqlServerConnectionConfig } from '@klo/connector-sqlserver';
|
||||
import {
|
||||
createDaemonLiveDatabaseIntrospection,
|
||||
createDefaultLocalIngestAdapters,
|
||||
type DefaultLocalIngestAdaptersOptions,
|
||||
type LiveDatabaseIntrospectionPort,
|
||||
LiveDatabaseSourceAdapter,
|
||||
type SourceAdapter,
|
||||
} from '@klo/context/ingest';
|
||||
import type { KloLocalProject } from '@klo/context/project';
|
||||
import { createHttpSqlAnalysisPort } from '@klo/context/sql-analysis';
|
||||
|
||||
function hasSnowflakeDriver(connection: unknown): boolean {
|
||||
return (
|
||||
typeof connection === 'object' &&
|
||||
connection !== null &&
|
||||
String((connection as { driver?: unknown }).driver ?? '').toLowerCase() === 'snowflake'
|
||||
);
|
||||
}
|
||||
|
||||
function createKloCliLiveDatabaseIntrospection(
|
||||
project: KloLocalProject,
|
||||
options: DefaultLocalIngestAdaptersOptions = {},
|
||||
): LiveDatabaseIntrospectionPort {
|
||||
const daemon = createDaemonLiveDatabaseIntrospection({
|
||||
connections: project.config.connections,
|
||||
...options.databaseIntrospection,
|
||||
...(options.databaseIntrospectionUrl ? { baseUrl: options.databaseIntrospectionUrl } : {}),
|
||||
});
|
||||
const sqlite = createSqliteLiveDatabaseIntrospection({
|
||||
projectDir: project.projectDir,
|
||||
connections: project.config.connections,
|
||||
});
|
||||
const mysql = createMysqlLiveDatabaseIntrospection({
|
||||
connections: project.config.connections,
|
||||
});
|
||||
const postgres = createPostgresLiveDatabaseIntrospection({
|
||||
connections: project.config.connections,
|
||||
});
|
||||
const clickhouse = createClickHouseLiveDatabaseIntrospection({
|
||||
connections: project.config.connections,
|
||||
});
|
||||
const sqlserver = createSqlServerLiveDatabaseIntrospection({
|
||||
connections: project.config.connections,
|
||||
});
|
||||
const bigquery = createBigQueryLiveDatabaseIntrospection({
|
||||
connections: project.config.connections,
|
||||
});
|
||||
return {
|
||||
async extractSchema(connectionId: string) {
|
||||
const connection = project.config.connections[connectionId];
|
||||
if (isKloPostgresConnectionConfig(connection)) {
|
||||
return postgres.extractSchema(connectionId);
|
||||
}
|
||||
if (isKloSqliteConnectionConfig(connection)) {
|
||||
return sqlite.extractSchema(connectionId);
|
||||
}
|
||||
if (isKloMysqlConnectionConfig(connection)) {
|
||||
return mysql.extractSchema(connectionId);
|
||||
}
|
||||
if (isKloClickHouseConnectionConfig(connection)) {
|
||||
return clickhouse.extractSchema(connectionId);
|
||||
}
|
||||
if (isKloSqlServerConnectionConfig(connection)) {
|
||||
return sqlserver.extractSchema(connectionId);
|
||||
}
|
||||
if (isKloBigQueryConnectionConfig(connection)) {
|
||||
return bigquery.extractSchema(connectionId);
|
||||
}
|
||||
if (hasSnowflakeDriver(connection)) {
|
||||
const { createSnowflakeLiveDatabaseIntrospection, isKloSnowflakeConnectionConfig } = await import(
|
||||
'@klo/connector-snowflake'
|
||||
);
|
||||
if (!isKloSnowflakeConnectionConfig(connection)) {
|
||||
return daemon.extractSchema(connectionId);
|
||||
}
|
||||
const snowflake = createSnowflakeLiveDatabaseIntrospection({
|
||||
connections: project.config.connections,
|
||||
});
|
||||
return snowflake.extractSchema(connectionId);
|
||||
}
|
||||
return daemon.extractSchema(connectionId);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
interface KloCliLocalIngestAdaptersOptions extends DefaultLocalIngestAdaptersOptions {
|
||||
historicSqlConnectionId?: string;
|
||||
sqlAnalysisUrl?: string;
|
||||
}
|
||||
|
||||
function isEnabledPostgresHistoricSqlConnection(connection: KloPostgresConnectionConfig | undefined): boolean {
|
||||
if (!connection || !isKloPostgresConnectionConfig(connection)) {
|
||||
return false;
|
||||
}
|
||||
const historicSql =
|
||||
typeof connection.historicSql === 'object' &&
|
||||
connection.historicSql !== null &&
|
||||
!Array.isArray(connection.historicSql)
|
||||
? (connection.historicSql as Record<string, unknown>)
|
||||
: null;
|
||||
return historicSql?.enabled === true && historicSql.dialect === 'postgres';
|
||||
}
|
||||
|
||||
function createEphemeralPostgresHistoricSqlClient(project: KloLocalProject, connectionId: string) {
|
||||
const connection = project.config.connections[connectionId] as KloPostgresConnectionConfig | undefined;
|
||||
if (!isKloPostgresConnectionConfig(connection)) {
|
||||
throw new Error(
|
||||
`Historic SQL local ingest requires a Postgres connection, got ${String(connection?.driver ?? 'unknown')}`,
|
||||
);
|
||||
}
|
||||
return {
|
||||
async executeQuery(sql: string, params?: unknown[]) {
|
||||
const client = new KloPostgresHistoricSqlQueryClient({
|
||||
connectionId,
|
||||
connection,
|
||||
});
|
||||
try {
|
||||
return await client.executeQuery(sql, params);
|
||||
} finally {
|
||||
await client.cleanup();
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function historicSqlOptionsForLocalRun(project: KloLocalProject, options: KloCliLocalIngestAdaptersOptions) {
|
||||
const connectionId = options.historicSqlConnectionId;
|
||||
if (!connectionId) {
|
||||
return undefined;
|
||||
}
|
||||
const connection = project.config.connections[connectionId] as KloPostgresConnectionConfig | undefined;
|
||||
if (!isEnabledPostgresHistoricSqlConnection(connection)) {
|
||||
return undefined;
|
||||
}
|
||||
return {
|
||||
sqlAnalysis: createHttpSqlAnalysisPort({
|
||||
baseUrl:
|
||||
options.sqlAnalysisUrl ??
|
||||
process.env.KLO_SQL_ANALYSIS_URL ??
|
||||
process.env.KLO_DAEMON_URL ??
|
||||
'http://127.0.0.1:8765',
|
||||
}),
|
||||
postgresQueryClient: createEphemeralPostgresHistoricSqlClient(project, connectionId),
|
||||
postgresBaselineRootDir: join(project.projectDir, '.klo/cache/historic-sql'),
|
||||
};
|
||||
}
|
||||
|
||||
export function createKloCliLocalIngestAdapters(
|
||||
project: KloLocalProject,
|
||||
options: KloCliLocalIngestAdaptersOptions = {},
|
||||
): SourceAdapter[] {
|
||||
const historicSql = historicSqlOptionsForLocalRun(project, options);
|
||||
const base = createDefaultLocalIngestAdapters(project, {
|
||||
...options,
|
||||
...(historicSql ? { historicSql } : {}),
|
||||
});
|
||||
const liveDatabase = new LiveDatabaseSourceAdapter({
|
||||
introspection: createKloCliLiveDatabaseIntrospection(project, options),
|
||||
});
|
||||
return base.map((adapter) => (adapter.source === 'live-database' ? liveDatabase : adapter));
|
||||
}
|
||||
163
packages/cli/src/local-scan-connectors.test.ts
Normal file
163
packages/cli/src/local-scan-connectors.test.ts
Normal file
|
|
@ -0,0 +1,163 @@
|
|||
import { mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { initKloProject, loadKloProject } from '@klo/context/project';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { createKloCliScanConnector } from './local-scan-connectors.js';
|
||||
|
||||
const bigQueryMock = vi.hoisted(() => ({
|
||||
constructorInputs: [] as Array<{
|
||||
connectionId: string;
|
||||
connection: unknown;
|
||||
maxBytesBilled?: number | string;
|
||||
}>,
|
||||
}));
|
||||
|
||||
vi.mock('@klo/connector-bigquery', () => ({
|
||||
isKloBigQueryConnectionConfig: (connection: { driver?: unknown } | undefined) =>
|
||||
String(connection?.driver ?? '').toLowerCase() === 'bigquery',
|
||||
KloBigQueryScanConnector: class {
|
||||
readonly id: string;
|
||||
readonly driver = 'bigquery';
|
||||
|
||||
constructor(options: { connectionId: string; connection: unknown; maxBytesBilled?: number | string }) {
|
||||
bigQueryMock.constructorInputs.push(options);
|
||||
this.id = `bigquery:${options.connectionId}`;
|
||||
}
|
||||
},
|
||||
}));
|
||||
|
||||
describe('createKloCliScanConnector', () => {
|
||||
let tempDir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
bigQueryMock.constructorInputs.length = 0;
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'klo-cli-scan-connector-'));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(tempDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('creates a native sqlite connector from standalone config', async () => {
|
||||
await initKloProject({ projectDir: tempDir, projectName: 'warehouse' });
|
||||
await writeFile(
|
||||
join(tempDir, 'klo.yaml'),
|
||||
[
|
||||
'project: warehouse',
|
||||
'connections:',
|
||||
' warehouse:',
|
||||
' driver: sqlite',
|
||||
' path: warehouse.db',
|
||||
' readonly: true',
|
||||
'',
|
||||
].join('\n'),
|
||||
'utf-8',
|
||||
);
|
||||
const project = await loadKloProject({ projectDir: tempDir });
|
||||
|
||||
const connector = await createKloCliScanConnector(project, 'warehouse');
|
||||
|
||||
expect(connector.id).toBe('sqlite:warehouse');
|
||||
expect(connector.driver).toBe('sqlite');
|
||||
});
|
||||
|
||||
it.each([
|
||||
['maxBytesBilled', ' maxBytesBilled: 123456789', 123456789],
|
||||
['max_bytes_billed', ' max_bytes_billed: "987654321"', '987654321'],
|
||||
])('passes BigQuery %s from standalone config', async (_label, byteCapLine, expectedMaxBytesBilled) => {
|
||||
await initKloProject({ projectDir: tempDir, projectName: 'warehouse' });
|
||||
await writeFile(
|
||||
join(tempDir, 'klo.yaml'),
|
||||
[
|
||||
'project: warehouse',
|
||||
'connections:',
|
||||
' warehouse:',
|
||||
' driver: bigquery',
|
||||
' dataset_id: analytics',
|
||||
' readonly: true',
|
||||
byteCapLine,
|
||||
'',
|
||||
].join('\n'),
|
||||
'utf-8',
|
||||
);
|
||||
const project = await loadKloProject({ projectDir: tempDir });
|
||||
|
||||
const connector = await createKloCliScanConnector(project, 'warehouse');
|
||||
|
||||
expect(connector.id).toBe('bigquery:warehouse');
|
||||
expect(connector.driver).toBe('bigquery');
|
||||
expect(bigQueryMock.constructorInputs).toEqual([
|
||||
expect.objectContaining({
|
||||
connectionId: 'warehouse',
|
||||
maxBytesBilled: expectedMaxBytesBilled,
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
it('does not create a standalone PostHog scan connector', async () => {
|
||||
await initKloProject({ projectDir: tempDir, projectName: 'warehouse' });
|
||||
await writeFile(
|
||||
join(tempDir, 'klo.yaml'),
|
||||
[
|
||||
'project: warehouse',
|
||||
'connections:',
|
||||
' product:',
|
||||
' driver: posthog',
|
||||
' api_key: phx_test',
|
||||
' project_id: "157881"',
|
||||
' readonly: true',
|
||||
'',
|
||||
].join('\n'),
|
||||
'utf-8',
|
||||
);
|
||||
const project = await loadKloProject({ projectDir: tempDir });
|
||||
|
||||
await expect(createKloCliScanConnector(project, 'product')).rejects.toThrow(
|
||||
'Connection "product" uses driver "posthog", which has no native standalone KLO scan connector',
|
||||
);
|
||||
});
|
||||
|
||||
it('throws for structural daemon-only fallback configs', async () => {
|
||||
await initKloProject({ projectDir: tempDir, projectName: 'warehouse' });
|
||||
await writeFile(
|
||||
join(tempDir, 'klo.yaml'),
|
||||
[
|
||||
'project: warehouse',
|
||||
'connections:',
|
||||
' warehouse:',
|
||||
' driver: duckdb',
|
||||
' path: warehouse.duckdb',
|
||||
'',
|
||||
].join('\n'),
|
||||
'utf-8',
|
||||
);
|
||||
const project = await loadKloProject({ projectDir: tempDir });
|
||||
|
||||
await expect(createKloCliScanConnector(project, 'warehouse')).rejects.toThrow(
|
||||
'Connection "warehouse" uses driver "duckdb", which has no native standalone KLO scan connector',
|
||||
);
|
||||
});
|
||||
|
||||
it('throws a clear error when the connection block has no driver field', async () => {
|
||||
await initKloProject({ projectDir: tempDir, projectName: 'warehouse' });
|
||||
await writeFile(
|
||||
join(tempDir, 'klo.yaml'),
|
||||
[
|
||||
'project: warehouse',
|
||||
'connections:',
|
||||
' warehouse:',
|
||||
' type: postgres',
|
||||
' url: postgresql://example/db',
|
||||
' readonly: true',
|
||||
'',
|
||||
].join('\n'),
|
||||
'utf-8',
|
||||
);
|
||||
const project = await loadKloProject({ projectDir: tempDir });
|
||||
|
||||
await expect(createKloCliScanConnector(project, 'warehouse')).rejects.toThrow(
|
||||
'Connection "warehouse" has no `driver` field in klo.yaml',
|
||||
);
|
||||
});
|
||||
});
|
||||
84
packages/cli/src/local-scan-connectors.ts
Normal file
84
packages/cli/src/local-scan-connectors.ts
Normal file
|
|
@ -0,0 +1,84 @@
|
|||
import type { KloLocalProject } from '@klo/context/project';
|
||||
import type { KloScanConnector } from '@klo/context/scan';
|
||||
|
||||
const SUPPORTED_DRIVERS = 'sqlite, postgres, mysql, clickhouse, sqlserver, bigquery, snowflake';
|
||||
|
||||
function bigQueryMaxBytesBilled(
|
||||
connection: KloLocalProject['config']['connections'][string],
|
||||
): number | string | undefined {
|
||||
const raw = connection.maxBytesBilled ?? connection.max_bytes_billed;
|
||||
if (typeof raw === 'number') {
|
||||
return Number.isFinite(raw) && raw > 0 ? raw : undefined;
|
||||
}
|
||||
if (typeof raw === 'string') {
|
||||
const trimmed = raw.trim();
|
||||
return trimmed.length > 0 ? trimmed : undefined;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export async function createKloCliScanConnector(
|
||||
project: KloLocalProject,
|
||||
connectionId: string,
|
||||
): Promise<KloScanConnector> {
|
||||
const connection = project.config.connections[connectionId];
|
||||
if (!connection) {
|
||||
throw new Error(`Connection "${connectionId}" is not configured in klo.yaml`);
|
||||
}
|
||||
const driver = String(connection.driver ?? '').toLowerCase();
|
||||
if (!driver) {
|
||||
throw new Error(
|
||||
`Connection "${connectionId}" has no \`driver\` field in klo.yaml. Supported drivers: ${SUPPORTED_DRIVERS}.`,
|
||||
);
|
||||
}
|
||||
if (driver === 'sqlite' || driver === 'sqlite3') {
|
||||
const { KloSqliteScanConnector, isKloSqliteConnectionConfig } = await import('@klo/connector-sqlite');
|
||||
if (isKloSqliteConnectionConfig(connection)) {
|
||||
return new KloSqliteScanConnector({ connectionId, connection, projectDir: project.projectDir });
|
||||
}
|
||||
}
|
||||
if (driver === 'postgres' || driver === 'postgresql') {
|
||||
const { KloPostgresScanConnector, isKloPostgresConnectionConfig } = await import('@klo/connector-postgres');
|
||||
if (isKloPostgresConnectionConfig(connection)) {
|
||||
return new KloPostgresScanConnector({ connectionId, connection });
|
||||
}
|
||||
}
|
||||
if (driver === 'mysql') {
|
||||
const { KloMysqlScanConnector, isKloMysqlConnectionConfig } = await import('@klo/connector-mysql');
|
||||
if (isKloMysqlConnectionConfig(connection)) {
|
||||
return new KloMysqlScanConnector({ connectionId, connection });
|
||||
}
|
||||
}
|
||||
if (driver === 'clickhouse') {
|
||||
const { KloClickHouseScanConnector, isKloClickHouseConnectionConfig } = await import('@klo/connector-clickhouse');
|
||||
if (isKloClickHouseConnectionConfig(connection)) {
|
||||
return new KloClickHouseScanConnector({ connectionId, connection });
|
||||
}
|
||||
}
|
||||
if (driver === 'sqlserver') {
|
||||
const { KloSqlServerScanConnector, isKloSqlServerConnectionConfig } = await import('@klo/connector-sqlserver');
|
||||
if (isKloSqlServerConnectionConfig(connection)) {
|
||||
return new KloSqlServerScanConnector({ connectionId, connection });
|
||||
}
|
||||
}
|
||||
if (driver === 'bigquery') {
|
||||
const { KloBigQueryScanConnector, isKloBigQueryConnectionConfig } = await import('@klo/connector-bigquery');
|
||||
if (isKloBigQueryConnectionConfig(connection)) {
|
||||
const maxBytesBilled = bigQueryMaxBytesBilled(connection);
|
||||
return new KloBigQueryScanConnector({
|
||||
connectionId,
|
||||
connection,
|
||||
...(maxBytesBilled !== undefined ? { maxBytesBilled } : {}),
|
||||
});
|
||||
}
|
||||
}
|
||||
if (driver === 'snowflake') {
|
||||
const { KloSnowflakeScanConnector, isKloSnowflakeConnectionConfig } = await import('@klo/connector-snowflake');
|
||||
if (isKloSnowflakeConnectionConfig(connection)) {
|
||||
return new KloSnowflakeScanConnector({ connectionId, connection });
|
||||
}
|
||||
}
|
||||
throw new Error(
|
||||
`Connection "${connectionId}" uses driver "${driver}", which has no native standalone KLO scan connector. Supported drivers: ${SUPPORTED_DRIVERS}.`,
|
||||
);
|
||||
}
|
||||
597
packages/cli/src/memory-flow-hud.tsx
Normal file
597
packages/cli/src/memory-flow-hud.tsx
Normal file
|
|
@ -0,0 +1,597 @@
|
|||
/* @jsxImportSource react */
|
||||
import type { MemoryFlowEvent, MemoryFlowReplayInput } from '@klo/context/ingest/memory-flow';
|
||||
import { Box, Text } from 'ink';
|
||||
import React, { type ReactNode } from 'react';
|
||||
import { buildDemoMetrics, formatCost, formatDuration } from './demo-metrics.js';
|
||||
import { formatNextStepLines } from './next-steps.js';
|
||||
import { profileMark } from './startup-profile.js';
|
||||
|
||||
profileMark('module:memory-flow-hud');
|
||||
|
||||
interface HudTheme {
|
||||
text: string;
|
||||
muted: string;
|
||||
active: string;
|
||||
complete: string;
|
||||
warning: string;
|
||||
failed: string;
|
||||
border: string;
|
||||
}
|
||||
|
||||
const SPINNER_FRAMES = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'] as const;
|
||||
|
||||
function spinner(frame: number): string {
|
||||
return SPINNER_FRAMES[frame % SPINNER_FRAMES.length] ?? '⠋';
|
||||
}
|
||||
|
||||
function counterValue(target: number, frame: number, framesToFill = 12): number {
|
||||
if (target <= 0 || frame <= 0) return 0;
|
||||
if (frame >= framesToFill) return target;
|
||||
return Math.round((frame / framesToFill) * target);
|
||||
}
|
||||
|
||||
function hasWorkStarted(input: MemoryFlowReplayInput): boolean {
|
||||
return input.events.some((e) => e.type === 'work_unit_started');
|
||||
}
|
||||
|
||||
function isPrepopulatedDemoReplay(input: MemoryFlowReplayInput): boolean {
|
||||
return input.metadata?.origin === 'packaged' || input.metadata?.timing === 'prebuilt';
|
||||
}
|
||||
|
||||
function flowLine(width: number, frame: number, active: boolean): string {
|
||||
if (!active) return '━'.repeat(width);
|
||||
const pulse = ['░', '▒', '▓', '█', '█', '█', '▓', '▒', '░'];
|
||||
const pw = pulse.length;
|
||||
const chars: string[] = [];
|
||||
const offset = (frame * 2) % (width + pw);
|
||||
for (let i = 0; i < width; i += 1) {
|
||||
const p = i - offset + pw;
|
||||
chars.push(p >= 0 && p < pw ? (pulse[p] ?? '━') : '━');
|
||||
}
|
||||
return chars.join('');
|
||||
}
|
||||
|
||||
function brailleFlow(width: number, frame: number): string {
|
||||
// Braille unicode: U+2800 + dot bitmask
|
||||
// Dots: 1=0x01 2=0x02 3=0x04 4=0x08 5=0x10 6=0x20 7=0x40 8=0x80
|
||||
// Layout: col0=[1,2,3,7] col1=[4,5,6,8]
|
||||
const chars: string[] = [];
|
||||
for (let i = 0; i < width; i += 1) {
|
||||
const density = (i + 1) / width;
|
||||
const phase = (i * 3 + frame * 2) % 12;
|
||||
let dots = 0;
|
||||
|
||||
// Sparse diagonal streams on the left, dense on the right
|
||||
// Each "stream" is a diagonal line of dots moving rightward
|
||||
if ((phase + 0) % 4 < density * 4) dots |= 0x01; // dot 1
|
||||
if ((phase + 1) % 5 < density * 4) dots |= 0x08; // dot 4
|
||||
if ((phase + 2) % 4 < density * 3) dots |= 0x02; // dot 2
|
||||
if ((phase + 3) % 5 < density * 3) dots |= 0x10; // dot 5
|
||||
if ((phase + 4) % 4 < density * 2.5) dots |= 0x04; // dot 3
|
||||
if ((phase + 5) % 5 < density * 2.5) dots |= 0x20; // dot 6
|
||||
if ((phase + 1) % 6 < density * 2) dots |= 0x40; // dot 7
|
||||
if ((phase + 3) % 6 < density * 2) dots |= 0x80; // dot 8
|
||||
|
||||
chars.push(String.fromCharCode(0x2800 + dots));
|
||||
}
|
||||
return chars.join('');
|
||||
}
|
||||
|
||||
function progressBarOverall(
|
||||
finishedCount: number,
|
||||
activeCount: number,
|
||||
totalCount: number,
|
||||
width: number,
|
||||
frame: number,
|
||||
): string {
|
||||
if (totalCount === 0) return '░'.repeat(width);
|
||||
|
||||
const finishedWidth = Math.round((finishedCount / totalCount) * width);
|
||||
const activeWidth = Math.max(activeCount > 0 ? 1 : 0, Math.round((activeCount / totalCount) * width));
|
||||
const queuedWidth = Math.max(0, width - finishedWidth - activeWidth);
|
||||
|
||||
const finished = '█'.repeat(finishedWidth);
|
||||
|
||||
const pulse = ['░', '▒', '▓', '█', '▓', '▒'];
|
||||
const pulseLen = pulse.length;
|
||||
const offset = (frame * 2) % (activeWidth + pulseLen);
|
||||
const activeChars: string[] = [];
|
||||
for (let i = 0; i < activeWidth; i += 1) {
|
||||
const p = i - offset + pulseLen;
|
||||
activeChars.push(p >= 0 && p < pulseLen ? (pulse[p] ?? '▒') : '▒');
|
||||
}
|
||||
|
||||
return finished + activeChars.join('') + '░'.repeat(queuedWidth);
|
||||
}
|
||||
|
||||
function sparkleWipe(width: number, frame: number, row: number): string {
|
||||
const chars: string[] = [];
|
||||
const sweepPos = (frame * 2 + row * 6) % (width + 8);
|
||||
const sparkles = ['✨', '✦', '✧', '·'];
|
||||
for (let i = 0; i < width; i += 1) {
|
||||
const dist = i - sweepPos;
|
||||
if (dist < -6) {
|
||||
const t = (i * 11 + row * 5 + frame * 3) % 10;
|
||||
chars.push(t === 0 ? sparkles[0]! : t === 3 ? sparkles[1]! : t === 7 ? sparkles[2]! : ' ');
|
||||
} else if (dist < -3) {
|
||||
const t = (i + frame) % 3;
|
||||
chars.push(t === 0 ? sparkles[1]! : t === 1 ? sparkles[2]! : sparkles[3]!);
|
||||
} else if (dist <= 0) {
|
||||
const gradient = ['░', '▒', '▓', '█'];
|
||||
chars.push(gradient[Math.min(3, dist + 3)] ?? '█');
|
||||
} else if (dist <= 2) {
|
||||
chars.push(dist === 1 ? '▓' : '▒');
|
||||
} else {
|
||||
const noise = (i * 31 + row * 17 + frame * 3) % 5;
|
||||
const messy = ['░', '▒', '▓', '▒', '░'];
|
||||
chars.push(messy[noise] ?? '▒');
|
||||
}
|
||||
}
|
||||
return chars.join('');
|
||||
}
|
||||
|
||||
function activityWave(width: number, frame: number, offset: number): string {
|
||||
const heights = ['▁', '▂', '▃', '▄', '▅', '▆', '▇', '█'];
|
||||
const chars: string[] = [];
|
||||
for (let i = 0; i < width; i += 1) {
|
||||
const wave = Math.sin(((i * 2 + frame + offset * 5) * Math.PI) / 6);
|
||||
const idx = Math.round(((wave + 1) / 2) * (heights.length - 1));
|
||||
chars.push(heights[idx] ?? '▁');
|
||||
}
|
||||
return chars.join('');
|
||||
}
|
||||
|
||||
function topicName(key: string): string {
|
||||
return (key.split('/').pop()?.replace(/\.md$/, '') ?? key).replace(/[_-]/g, ' ');
|
||||
}
|
||||
|
||||
function tableName(key: string): string {
|
||||
return key.split('.').pop()?.replace(/[_-]/g, ' ') ?? key;
|
||||
}
|
||||
|
||||
function humanizeInsight(key: string, target: 'sl' | 'wiki', summary: string | undefined): string {
|
||||
if (summary) return summary;
|
||||
const name = target === 'sl' ? tableName(key) : topicName(key);
|
||||
return target === 'sl' ? `Query definition: ${name}` : `Knowledge page: ${name}`;
|
||||
}
|
||||
|
||||
const ADAPTER_PREFIXES = ['live_database_', 'metabase_', 'looker_', 'lookml_', 'metricflow_', 'notion_', 'historic_sql_', 'dbt_descriptions_'];
|
||||
const INTERNAL_DEMO_CONNECTION_ID = 'orbit_demo';
|
||||
const PUBLIC_DEMO_SOURCE_LABEL = 'Orbit Demo';
|
||||
|
||||
function humanizeUnitKey(unitKey: string): string {
|
||||
let key = unitKey.replace(/-/g, '_');
|
||||
for (const prefix of ADAPTER_PREFIXES) {
|
||||
if (key.startsWith(prefix)) { key = key.slice(prefix.length); break; }
|
||||
}
|
||||
return key.replace(/_/g, ' ');
|
||||
}
|
||||
|
||||
interface SourceInfo {
|
||||
type: string;
|
||||
name: string;
|
||||
sourceCount: string;
|
||||
itemNounPlural: string;
|
||||
readingVerb: string;
|
||||
ingestDescription: string;
|
||||
}
|
||||
|
||||
const ADAPTER_LABELS: Record<string, { type: string; plural: string; verb: string; description: string }> = {
|
||||
'live-database': { type: 'Database', plural: 'tables', verb: 'Reading', description: 'Reading table schemas, understanding relationships, creating query definitions' },
|
||||
metricflow: { type: 'dbt project', plural: 'models', verb: 'Parsing', description: 'Parsing dbt models, extracting metric definitions, mapping dependencies' },
|
||||
looker: { type: 'Looker', plural: 'explores', verb: 'Analyzing', description: 'Analyzing explores, extracting dimensions and measures, mapping joins' },
|
||||
lookml: { type: 'LookML', plural: 'views', verb: 'Parsing', description: 'Parsing LookML views, extracting field definitions, mapping relationships' },
|
||||
metabase: { type: 'Metabase', plural: 'questions', verb: 'Analyzing', description: 'Analyzing saved questions, extracting query patterns, understanding dashboards' },
|
||||
notion: { type: 'Notion', plural: 'pages', verb: 'Reading', description: 'Reading pages, extracting structure, understanding your documentation' },
|
||||
'historic-sql': { type: 'SQL history', plural: 'queries', verb: 'Analyzing', description: 'Analyzing query patterns, identifying common joins, learning access patterns' },
|
||||
'dbt-descriptions': { type: 'dbt schema', plural: 'models', verb: 'Parsing', description: 'Parsing schema definitions, extracting descriptions, mapping lineage' },
|
||||
dbt_descriptions: { type: 'dbt', plural: 'models', verb: 'Parsing', description: 'Parsing schema definitions, extracting descriptions, mapping lineage' },
|
||||
};
|
||||
|
||||
function sourceDescription(input: MemoryFlowReplayInput): SourceInfo {
|
||||
const adapter = input.adapter ?? 'source';
|
||||
const conn = input.connectionId ?? '';
|
||||
const sourceEvents = input.events.filter((e) => e.type === 'source_acquired') as Array<{ type: 'source_acquired'; adapter: string; fileCount: number }>;
|
||||
const isDemoSource = conn === INTERNAL_DEMO_CONNECTION_ID || isPrepopulatedDemoReplay(input);
|
||||
|
||||
if (isDemoSource && sourceEvents.length <= 1) {
|
||||
const count = sourceEvents[0] ? String(sourceEvents[0].fileCount) : '?';
|
||||
return {
|
||||
type: PUBLIC_DEMO_SOURCE_LABEL,
|
||||
name: '',
|
||||
sourceCount: count,
|
||||
itemNounPlural: 'sources',
|
||||
readingVerb: 'Ingesting',
|
||||
ingestDescription: 'Ingesting warehouse, dbt, BI, and docs into a unified context layer',
|
||||
};
|
||||
}
|
||||
|
||||
if (sourceEvents.length > 1) {
|
||||
const totalFiles = sourceEvents.reduce((sum, s) => sum + s.fileCount, 0);
|
||||
const labels = [...new Set(sourceEvents.map((s) => ADAPTER_LABELS[s.adapter]?.type ?? s.adapter))];
|
||||
return {
|
||||
type: labels.join(' + '),
|
||||
name: conn,
|
||||
sourceCount: String(totalFiles),
|
||||
itemNounPlural: 'sources',
|
||||
readingVerb: 'Ingesting',
|
||||
ingestDescription: 'Ingesting warehouse, dbt, BI, and docs into a unified context layer',
|
||||
};
|
||||
}
|
||||
|
||||
const count = sourceEvents[0] ? String(sourceEvents[0].fileCount) : '?';
|
||||
const info = ADAPTER_LABELS[adapter] ?? { type: adapter, plural: 'sources', verb: 'Reading', description: 'Reading sources, understanding structure, creating definitions' };
|
||||
return { type: info.type, name: conn, sourceCount: count, itemNounPlural: info.plural, readingVerb: info.verb, ingestDescription: info.description };
|
||||
}
|
||||
|
||||
function activeWorkUnit(
|
||||
input: MemoryFlowReplayInput,
|
||||
): { unitKey: string; stepIndex: number; stepBudget: number } | null {
|
||||
const units = activeWorkUnits(input);
|
||||
return units.at(-1) ?? null;
|
||||
}
|
||||
|
||||
function activeWorkUnits(
|
||||
input: MemoryFlowReplayInput,
|
||||
): Array<{ unitKey: string; stepIndex: number; stepBudget: number }> {
|
||||
const finishedKeys = new Set<string>();
|
||||
const unitMap = new Map<string, { stepIndex: number; stepBudget: number }>();
|
||||
|
||||
for (const e of input.events) {
|
||||
if (e.type === 'work_unit_started') {
|
||||
unitMap.set(e.unitKey, { stepIndex: 0, stepBudget: e.stepBudget });
|
||||
}
|
||||
if (e.type === 'work_unit_step') {
|
||||
const existing = unitMap.get(e.unitKey);
|
||||
if (existing) {
|
||||
existing.stepIndex = e.stepIndex;
|
||||
existing.stepBudget = e.stepBudget;
|
||||
}
|
||||
}
|
||||
if (e.type === 'work_unit_finished') finishedKeys.add(e.unitKey);
|
||||
}
|
||||
|
||||
const result: Array<{ unitKey: string; stepIndex: number; stepBudget: number }> = [];
|
||||
for (const [unitKey, data] of unitMap) {
|
||||
if (!finishedKeys.has(unitKey)) result.push({ unitKey, ...data });
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function queuedWorkUnits(input: MemoryFlowReplayInput): string[] {
|
||||
const startedKeys = new Set<string>();
|
||||
for (const e of input.events) {
|
||||
if (e.type === 'work_unit_started') startedKeys.add(e.unitKey);
|
||||
}
|
||||
return input.plannedWorkUnits.filter((u) => !startedKeys.has(u.unitKey)).map((u) => u.unitKey);
|
||||
}
|
||||
|
||||
interface Insight {
|
||||
icon: string;
|
||||
text: string;
|
||||
unitKey: string;
|
||||
hasSummary: boolean;
|
||||
}
|
||||
|
||||
function buildInsights(input: MemoryFlowReplayInput): Insight[] {
|
||||
return input.events
|
||||
.filter((e) => e.type === 'candidate_action')
|
||||
.map((e) => {
|
||||
const ca = e as { unitKey: string; target: 'sl' | 'wiki'; key: string };
|
||||
const detail = input.details.actions.find((a) => a.key === ca.key && a.unitKey === ca.unitKey);
|
||||
return {
|
||||
icon: ca.target === 'sl' ? '📊' : '📝',
|
||||
text: humanizeInsight(ca.key, ca.target, detail?.summary),
|
||||
unitKey: ca.unitKey,
|
||||
hasSummary: !!detail?.summary,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function finishedUnits(input: MemoryFlowReplayInput): Array<{ unitKey: string; artifactCount: number }> {
|
||||
const units: Array<{ unitKey: string; artifactCount: number }> = [];
|
||||
for (const e of input.events) {
|
||||
if (e.type === 'work_unit_finished' && e.status === 'success') {
|
||||
const count = input.events.filter((a) => a.type === 'candidate_action' && a.unitKey === e.unitKey).length;
|
||||
units.push({ unitKey: e.unitKey, artifactCount: count });
|
||||
}
|
||||
}
|
||||
return units;
|
||||
}
|
||||
|
||||
function artifactCounts(input: MemoryFlowReplayInput): { sl: number; wiki: number } {
|
||||
let sl = 0;
|
||||
let wiki = 0;
|
||||
for (const e of input.events) {
|
||||
if (e.type === 'candidate_action') {
|
||||
if (e.target === 'sl') sl++;
|
||||
else wiki++;
|
||||
}
|
||||
}
|
||||
return { sl, wiki };
|
||||
}
|
||||
|
||||
function pad(str: string, width: number): string {
|
||||
return str.length >= width ? str : str + ' '.repeat(width - str.length);
|
||||
}
|
||||
|
||||
const KLO_LOGO_SMALL = [
|
||||
'██╗ ██╗██╗ ██████╗ ',
|
||||
'██║ ██╔╝██║ ██╔═══██╗',
|
||||
'█████╔╝ ██║ ██║ ██║',
|
||||
'██╔═██╗ ██║ ██║ ██║',
|
||||
'██║ ██╗███████╗╚██████╔╝',
|
||||
'╚═╝ ╚═╝╚══════╝ ╚═════╝ ',
|
||||
] as const;
|
||||
|
||||
export function Logo(props: { theme: HudTheme; done: boolean }): ReactNode {
|
||||
const color = props.done ? props.theme.complete : props.theme.active;
|
||||
return (
|
||||
<Box flexDirection="column" marginBottom={1} paddingLeft={2}>
|
||||
{KLO_LOGO_SMALL.map((line, idx) => (
|
||||
<Text key={idx} color={color}>
|
||||
{line}
|
||||
</Text>
|
||||
))}
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
|
||||
export function Hud(props: {
|
||||
input: MemoryFlowReplayInput;
|
||||
theme: HudTheme;
|
||||
frame: number;
|
||||
width: number;
|
||||
now?: () => number;
|
||||
}): ReactNode {
|
||||
const isRunning = props.input.status === 'running';
|
||||
const isDone = props.input.status === 'done';
|
||||
const isFlowing = isRunning && hasWorkStarted(props.input);
|
||||
|
||||
const src = sourceDescription(props.input);
|
||||
const counts = artifactCounts(props.input);
|
||||
const metrics = buildDemoMetrics(props.input, props.now ? { now: props.now } : {});
|
||||
const workStarted = hasWorkStarted(props.input);
|
||||
|
||||
const sourceEvents = props.input.events.filter((e) => e.type === 'source_acquired');
|
||||
const col1Content = sourceEvents.length > 1 || !src.name ? src.type : `${src.type} (${src.name})`;
|
||||
|
||||
const innerWidth = Math.max(60, props.width - 6);
|
||||
|
||||
const actives = activeWorkUnits(props.input);
|
||||
const reconEvent = props.input.events.find((e) => e.type === 'reconciliation_finished');
|
||||
const allAnalyzed = isFlowing && actives.length === 0;
|
||||
const isReconciling = allAnalyzed && !reconEvent && !isDone;
|
||||
|
||||
const hLine = '─'.repeat(innerWidth);
|
||||
|
||||
const elapsed = formatDuration(metrics.elapsedMs);
|
||||
let eta = '';
|
||||
if (metrics.status === 'running' && metrics.etaMs !== null) eta = `~${formatDuration(metrics.etaMs)} left`;
|
||||
else if (metrics.status !== 'running') eta = 'done';
|
||||
const cost = workStarted ? formatCost(metrics.estimatedCostUsd) : '';
|
||||
const statsParts = [`⏱ ${elapsed}`, eta, cost].filter(Boolean).join(' ');
|
||||
const prepopulatedCostDisclaimer =
|
||||
cost && isPrepopulatedDemoReplay(props.input)
|
||||
? 'Pre-run demo: $ shown is illustrative; no money is being spent now.'
|
||||
: null;
|
||||
|
||||
return (
|
||||
<Box flexDirection="column" marginBottom={1}>
|
||||
<Text color={props.theme.border}> ╭{hLine}╮</Text>
|
||||
<Text>
|
||||
<Text color={props.theme.border}> │ </Text>
|
||||
<Text color={props.theme.text}>{col1Content}</Text>
|
||||
<Text color={props.theme.muted}> — {src.sourceCount} {src.itemNounPlural}</Text>
|
||||
</Text>
|
||||
<Text>
|
||||
<Text color={props.theme.border}> │ </Text>
|
||||
<Text color="#b8860b">{statsParts}</Text>
|
||||
</Text>
|
||||
{prepopulatedCostDisclaimer && (
|
||||
<Text>
|
||||
<Text color={props.theme.border}> │ </Text>
|
||||
<Text color={props.theme.muted}>{prepopulatedCostDisclaimer}</Text>
|
||||
</Text>
|
||||
)}
|
||||
<Text color={props.theme.border}> ╰{hLine}╯</Text>
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
|
||||
export function ActivityFeed(props: {
|
||||
input: MemoryFlowReplayInput;
|
||||
theme: HudTheme;
|
||||
frame: number;
|
||||
width: number;
|
||||
completionFrame: number;
|
||||
showCompletion: boolean;
|
||||
holdComplete: boolean;
|
||||
}): ReactNode {
|
||||
const actives = activeWorkUnits(props.input);
|
||||
const queued = queuedWorkUnits(props.input);
|
||||
const finished = finishedUnits(props.input);
|
||||
const insights = buildInsights(props.input);
|
||||
const src = sourceDescription(props.input);
|
||||
const isDone = props.input.status === 'done';
|
||||
const isError = props.input.status === 'error';
|
||||
|
||||
const diffEvent = props.input.events.find((e) => e.type === 'diff_computed') as
|
||||
| (MemoryFlowEvent & { added: number; modified: number; deleted: number; unchanged: number })
|
||||
| undefined;
|
||||
const planEvent = props.input.events.find((e) => e.type === 'chunks_planned') as
|
||||
| (MemoryFlowEvent & { chunkCount: number; workUnitCount: number })
|
||||
| undefined;
|
||||
const reconEvent = props.input.events.find((e) => e.type === 'reconciliation_finished') as
|
||||
| (MemoryFlowEvent & { conflictCount: number })
|
||||
| undefined;
|
||||
const savedEvent = props.input.events.find((e) => e.type === 'saved');
|
||||
|
||||
const workStarted = hasWorkStarted(props.input);
|
||||
const totalChunks = planEvent?.chunkCount ?? 0;
|
||||
const finishedWithArtifacts = finished.filter((u) => u.artifactCount > 0);
|
||||
const finishedAreas = totalChunks > 0 ? Math.min(finished.length, totalChunks) : finished.length;
|
||||
const allWorkDone = workStarted && actives.length === 0 && queued.length === 0;
|
||||
const isReconciling = allWorkDone && !reconEvent && !isDone && !isError;
|
||||
const isSaving = reconEvent && !savedEvent && !isDone && !isError;
|
||||
|
||||
const isIncremental = diffEvent && (diffEvent.modified > 0 || diffEvent.deleted > 0 || diffEvent.unchanged > 0);
|
||||
|
||||
const barWidth = Math.min(40, props.width - 20);
|
||||
|
||||
return (
|
||||
<Box flexDirection="column" paddingLeft={2}>
|
||||
{/* Phase 1: Connecting */}
|
||||
{!diffEvent && !workStarted && (
|
||||
<Text color={props.theme.active}>
|
||||
{spinner(props.frame)} Connecting to {src.type.toLowerCase()}...
|
||||
</Text>
|
||||
)}
|
||||
|
||||
{/* Phase 2: Connected */}
|
||||
{diffEvent && (
|
||||
<Text color={props.theme.complete}>
|
||||
✓ Connected — found {src.sourceCount} {src.itemNounPlural} to ingest
|
||||
</Text>
|
||||
)}
|
||||
|
||||
{/* Phase 2b: Diff (incremental runs only) */}
|
||||
{diffEvent && isIncremental && (
|
||||
<Text color={props.theme.complete}>
|
||||
✓ Compared with last sync — only re-analyzing what changed
|
||||
</Text>
|
||||
)}
|
||||
|
||||
{/* Phase 3: Planning */}
|
||||
{diffEvent && !planEvent && !workStarted && (
|
||||
<Text color={props.theme.active}>
|
||||
{spinner(props.frame)} Grouping related {src.itemNounPlural} together for deeper analysis...
|
||||
</Text>
|
||||
)}
|
||||
{planEvent && (
|
||||
<Text color={props.theme.complete}>
|
||||
✓ Grouped into {planEvent.chunkCount} business area{planEvent.chunkCount === 1 ? '' : 's'}
|
||||
</Text>
|
||||
)}
|
||||
|
||||
{/* Phase 4: Ingesting */}
|
||||
{workStarted && !allWorkDone && (
|
||||
<Box flexDirection="column" marginTop={1}>
|
||||
<Text color={props.theme.active}>
|
||||
{spinner(props.frame)} Ingesting — {finishedAreas}/{totalChunks || '?'} business area{totalChunks === 1 ? '' : 's'} done
|
||||
</Text>
|
||||
<Text color={props.theme.muted}>
|
||||
{' '}{src.ingestDescription}
|
||||
</Text>
|
||||
{totalChunks > 0 && (
|
||||
<Text color={props.theme.active}>
|
||||
{' '}
|
||||
{progressBarOverall(finishedAreas, actives.length, totalChunks, barWidth, props.frame)}
|
||||
</Text>
|
||||
)}
|
||||
</Box>
|
||||
)}
|
||||
|
||||
{/* Results — what KLO has created */}
|
||||
{insights.length > 0 && (
|
||||
<Box flexDirection="column" marginTop={1}>
|
||||
<Text color={props.theme.text}> Created so far:</Text>
|
||||
{insights.map((insight, idx) => (
|
||||
<Text key={`result-${idx}`} color={props.theme.muted}>
|
||||
{' '}{insight.icon} {insight.text}
|
||||
</Text>
|
||||
))}
|
||||
</Box>
|
||||
)}
|
||||
|
||||
{/* Phase 5: Finalizing */}
|
||||
{isReconciling && (
|
||||
<Text color={props.theme.active}>
|
||||
{spinner(props.frame)} Deduplicating — removing overlaps between business areas and checking for conflicts...
|
||||
</Text>
|
||||
)}
|
||||
{reconEvent && (
|
||||
<Text color={props.theme.complete}>
|
||||
✓ Deduplicated
|
||||
{reconEvent.conflictCount > 0
|
||||
? ` — ${reconEvent.conflictCount} conflict${reconEvent.conflictCount === 1 ? '' : 's'} resolved`
|
||||
: ' — no conflicts'}
|
||||
</Text>
|
||||
)}
|
||||
|
||||
{/* Phase 6: Saving */}
|
||||
{isSaving && (
|
||||
<Text color={props.theme.active}>{spinner(props.frame)} Saving to context layer...</Text>
|
||||
)}
|
||||
{savedEvent && (
|
||||
<Text color={props.theme.complete}>✓ Saved — your agents can now use the KLO context layer</Text>
|
||||
)}
|
||||
|
||||
{/* Phase 7: Completion */}
|
||||
{props.showCompletion && (isDone || isError) && (
|
||||
<CompletionSummary input={props.input} theme={props.theme} frame={props.completionFrame} holdComplete={props.holdComplete} />
|
||||
)}
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
|
||||
function CompletionSummary(props: {
|
||||
input: MemoryFlowReplayInput;
|
||||
theme: HudTheme;
|
||||
frame: number;
|
||||
holdComplete: boolean;
|
||||
}): ReactNode {
|
||||
const saved = [...props.input.events].reverse().find((e) => e.type === 'saved');
|
||||
const wikiCount = saved?.wikiCount ?? 0;
|
||||
const slCount = saved?.slCount ?? 0;
|
||||
const isError = props.input.status === 'error';
|
||||
|
||||
const sl = counterValue(slCount, props.frame);
|
||||
const wiki = counterValue(wikiCount, props.frame);
|
||||
|
||||
return (
|
||||
<Box flexDirection="column" marginTop={1}>
|
||||
{isError ? (
|
||||
<Text bold color={props.theme.failed}>
|
||||
✗ Something went wrong — review the errors above.
|
||||
</Text>
|
||||
) : (
|
||||
<>
|
||||
<Text color={props.theme.border}>{'─'.repeat(60)}</Text>
|
||||
<Text bold color={props.theme.complete}>
|
||||
★ KLO finished ingesting your data
|
||||
</Text>
|
||||
{(sl > 0 || wiki > 0) && (
|
||||
<>
|
||||
<Text />
|
||||
<Text color={props.theme.text}>KLO created:</Text>
|
||||
{sl > 0 && (
|
||||
<Text color={props.theme.active}>
|
||||
{' '}📊 {sl} query definition{sl === 1 ? '' : 's'} — so agents can write accurate SQL for your data
|
||||
</Text>
|
||||
)}
|
||||
{wiki > 0 && (
|
||||
<Text color={props.theme.complete}>
|
||||
{' '}📝 {wiki} knowledge page{wiki === 1 ? '' : 's'} — so agents understand your business context
|
||||
</Text>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
<Text />
|
||||
<Text color={props.theme.text}>What to do next:</Text>
|
||||
{formatNextStepLines().map((line) => (
|
||||
<Text key={line} color={props.theme.active}>
|
||||
{line}
|
||||
</Text>
|
||||
))}
|
||||
{props.holdComplete && (
|
||||
<>
|
||||
<Text />
|
||||
<Text color={props.theme.muted}>Press q to exit</Text>
|
||||
</>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
125
packages/cli/src/memory-flow-interactive.test.ts
Normal file
125
packages/cli/src/memory-flow-interactive.test.ts
Normal file
|
|
@ -0,0 +1,125 @@
|
|||
import { EventEmitter } from 'node:events';
|
||||
import type { MemoryFlowReplayInput } from '@klo/context/ingest';
|
||||
import { describe, expect, it, vi } from 'vitest';
|
||||
import { memoryFlowCommandForKey, renderMemoryFlowInteractively } from './memory-flow-interactive.js';
|
||||
|
||||
class FakeStdin extends EventEmitter {
|
||||
isTTY = true;
|
||||
isRaw = false;
|
||||
rawModes: boolean[] = [];
|
||||
resume = vi.fn();
|
||||
pause = vi.fn();
|
||||
|
||||
setRawMode(value: boolean): void {
|
||||
this.isRaw = value;
|
||||
this.rawModes.push(value);
|
||||
}
|
||||
}
|
||||
|
||||
function replay(): MemoryFlowReplayInput {
|
||||
return {
|
||||
runId: 'run-1',
|
||||
connectionId: 'warehouse',
|
||||
adapter: 'metricflow',
|
||||
status: 'done',
|
||||
sourceDir: '/tmp/source',
|
||||
syncId: 'sync-1',
|
||||
errors: [],
|
||||
plannedWorkUnits: [
|
||||
{
|
||||
unitKey: 'orders',
|
||||
rawFiles: ['models/orders.yml'],
|
||||
peerFileCount: 0,
|
||||
dependencyCount: 1,
|
||||
},
|
||||
{
|
||||
unitKey: 'customers',
|
||||
rawFiles: ['models/customers.yml'],
|
||||
peerFileCount: 0,
|
||||
dependencyCount: 0,
|
||||
},
|
||||
],
|
||||
details: { actions: [], provenance: [], transcripts: [] },
|
||||
events: [
|
||||
{ type: 'source_acquired', adapter: 'metricflow', trigger: 'manual_resync', fileCount: 2 },
|
||||
{ type: 'scope_detected', fingerprint: null },
|
||||
{ type: 'raw_snapshot_written', syncId: 'sync-1', rawFileCount: 2 },
|
||||
{ type: 'diff_computed', added: 1, modified: 1, deleted: 0, unchanged: 0 },
|
||||
{ type: 'chunks_planned', chunkCount: 2, workUnitCount: 2, evictionCount: 0 },
|
||||
{ type: 'work_unit_started', unitKey: 'orders', skills: ['knowledge_capture'], stepBudget: 4 },
|
||||
{ type: 'work_unit_finished', unitKey: 'orders', status: 'success' },
|
||||
{ type: 'work_unit_started', unitKey: 'customers', skills: ['knowledge_capture'], stepBudget: 4 },
|
||||
{ type: 'work_unit_finished', unitKey: 'customers', status: 'failed', reason: 'validation reset' },
|
||||
{ type: 'reconciliation_finished', conflictCount: 0, fallbackCount: 1 },
|
||||
{ type: 'saved', commitSha: 'abc12345', wikiCount: 1, slCount: 1 },
|
||||
{ type: 'provenance_recorded', rowCount: 2 },
|
||||
{ type: 'report_created', runId: 'run-1', reportPath: 'report-1' },
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
describe('memoryFlowCommandForKey', () => {
|
||||
it('maps supported terminal key names to memory-flow commands', () => {
|
||||
const idleSearch = { editing: false, query: '', matchIndex: 0 };
|
||||
const editingSearch = { editing: true, query: 'c', matchIndex: 0 };
|
||||
|
||||
expect(memoryFlowCommandForKey('', idleSearch, { name: 'left' })).toBe('left');
|
||||
expect(memoryFlowCommandForKey('', idleSearch, { name: 'right' })).toBe('right');
|
||||
expect(memoryFlowCommandForKey('', idleSearch, { name: 'up' })).toBe('up');
|
||||
expect(memoryFlowCommandForKey('', idleSearch, { name: 'down' })).toBe('down');
|
||||
expect(memoryFlowCommandForKey('', idleSearch, { name: 'return' })).toBe('enter');
|
||||
expect(memoryFlowCommandForKey('', idleSearch, { name: 'tab' })).toBe('tab');
|
||||
expect(memoryFlowCommandForKey('', idleSearch, { name: 'f' })).toBe('filter');
|
||||
expect(memoryFlowCommandForKey('', idleSearch, { name: 'p' })).toBe('provenance');
|
||||
expect(memoryFlowCommandForKey('', idleSearch, { name: 't' })).toBe('transcript');
|
||||
expect(memoryFlowCommandForKey('', idleSearch, { name: 'q' })).toBe('quit');
|
||||
expect(memoryFlowCommandForKey('', idleSearch, { name: 'c', ctrl: true })).toBe('quit');
|
||||
expect(memoryFlowCommandForKey('/', { editing: false, query: '', matchIndex: 0 }, { name: '/' })).toBe(
|
||||
'search-start',
|
||||
);
|
||||
expect(memoryFlowCommandForKey('c', { editing: true, query: '', matchIndex: 0 }, { name: 'c' })).toEqual({
|
||||
type: 'search-input',
|
||||
value: 'c',
|
||||
});
|
||||
expect(memoryFlowCommandForKey('', editingSearch, { name: 'backspace' })).toBe('search-backspace');
|
||||
expect(memoryFlowCommandForKey('', editingSearch, { name: 'return' })).toBe('search-submit');
|
||||
expect(memoryFlowCommandForKey('', editingSearch, { name: 'escape' })).toBe('search-clear');
|
||||
expect(memoryFlowCommandForKey('', idleSearch, { name: 'x' })).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('renderMemoryFlowInteractively', () => {
|
||||
it('repaints on keypress and restores raw mode on quit', async () => {
|
||||
let stdout = '';
|
||||
const stdin = new FakeStdin();
|
||||
const prepareKeypressEvents = vi.fn();
|
||||
|
||||
const promise = renderMemoryFlowInteractively(
|
||||
replay(),
|
||||
{
|
||||
stdin,
|
||||
stdout: {
|
||||
isTTY: true,
|
||||
columns: 120,
|
||||
write: (chunk) => {
|
||||
stdout += chunk;
|
||||
},
|
||||
},
|
||||
},
|
||||
{ prepareKeypressEvents },
|
||||
);
|
||||
|
||||
stdin.emit('keypress', '', { name: 'right' });
|
||||
stdin.emit('keypress', '', { name: 'tab' });
|
||||
stdin.emit('keypress', '', { name: 'q' });
|
||||
|
||||
await expect(promise).resolves.toBeUndefined();
|
||||
expect(prepareKeypressEvents).toHaveBeenCalledWith(stdin);
|
||||
expect(stdin.rawModes).toEqual([true, false]);
|
||||
expect(stdin.resume).toHaveBeenCalledTimes(1);
|
||||
expect(stdin.pause).toHaveBeenCalledTimes(1);
|
||||
expect(stdout).toContain('\u001b[2J\u001b[H');
|
||||
expect(stdout).toContain('[ACTIONS]');
|
||||
expect(stdout).toContain('Pane: trust');
|
||||
});
|
||||
});
|
||||
143
packages/cli/src/memory-flow-interactive.ts
Normal file
143
packages/cli/src/memory-flow-interactive.ts
Normal file
|
|
@ -0,0 +1,143 @@
|
|||
import { emitKeypressEvents } from 'node:readline';
|
||||
import {
|
||||
buildMemoryFlowViewModel,
|
||||
createInitialMemoryFlowInteractionState,
|
||||
reduceMemoryFlowInteractionState,
|
||||
renderMemoryFlowInteractive,
|
||||
type MemoryFlowInteractionCommand,
|
||||
type MemoryFlowInteractionState,
|
||||
type MemoryFlowReplayInput,
|
||||
} from '@klo/context/ingest';
|
||||
|
||||
interface KloMemoryFlowKey {
|
||||
name?: string;
|
||||
ctrl?: boolean;
|
||||
}
|
||||
|
||||
export interface KloMemoryFlowStdin {
|
||||
isTTY?: boolean;
|
||||
isRaw?: boolean;
|
||||
setRawMode?(value: boolean): void;
|
||||
resume?(): void;
|
||||
pause?(): void;
|
||||
on(event: 'keypress', listener: (chunk: string, key: KloMemoryFlowKey) => void): this;
|
||||
off?(event: 'keypress', listener: (chunk: string, key: KloMemoryFlowKey) => void): this;
|
||||
removeListener?(event: 'keypress', listener: (chunk: string, key: KloMemoryFlowKey) => void): this;
|
||||
}
|
||||
|
||||
interface KloMemoryFlowInteractiveIo {
|
||||
stdin?: KloMemoryFlowStdin;
|
||||
stdout: {
|
||||
isTTY?: boolean;
|
||||
columns?: number;
|
||||
write(chunk: string): void;
|
||||
};
|
||||
}
|
||||
|
||||
interface RenderMemoryFlowInteractiveOptions {
|
||||
prepareKeypressEvents?(stdin: KloMemoryFlowStdin): void;
|
||||
}
|
||||
|
||||
function defaultPrepareKeypressEvents(stdin: KloMemoryFlowStdin): void {
|
||||
emitKeypressEvents(stdin as Parameters<typeof emitKeypressEvents>[0]);
|
||||
}
|
||||
|
||||
export function memoryFlowCommandForKey(
|
||||
chunk: string,
|
||||
search: MemoryFlowInteractionState['search'],
|
||||
key: KloMemoryFlowKey,
|
||||
): MemoryFlowInteractionCommand | null {
|
||||
if (search.editing) {
|
||||
if (key.name === 'escape') return 'search-clear';
|
||||
if (key.name === 'return' || key.name === 'enter') return 'search-submit';
|
||||
if (key.name === 'backspace') return 'search-backspace';
|
||||
if (chunk.length === 1 && chunk >= ' ' && chunk !== '\u007f') {
|
||||
return { type: 'search-input', value: chunk };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
if (key.ctrl === true && key.name === 'c') {
|
||||
return 'quit';
|
||||
}
|
||||
|
||||
if (key.name === '/') return 'search-start';
|
||||
if (key.name === 'left') return 'left';
|
||||
if (key.name === 'right') return 'right';
|
||||
if (key.name === 'up') return 'up';
|
||||
if (key.name === 'down') return 'down';
|
||||
if (key.name === 'return' || key.name === 'enter') return 'enter';
|
||||
if (key.name === 'tab') return 'tab';
|
||||
if (key.name === 'f') return 'filter';
|
||||
if (key.name === 'p') return 'provenance';
|
||||
if (key.name === 't') return 'transcript';
|
||||
if (key.name === 'q' || key.name === 'escape') return 'quit';
|
||||
return null;
|
||||
}
|
||||
|
||||
function removeKeypressListener(
|
||||
stdin: KloMemoryFlowStdin,
|
||||
handler: (chunk: string, key: KloMemoryFlowKey) => void,
|
||||
): void {
|
||||
if (stdin.off) {
|
||||
stdin.off('keypress', handler);
|
||||
return;
|
||||
}
|
||||
stdin.removeListener?.('keypress', handler);
|
||||
}
|
||||
|
||||
function repaint(input: MemoryFlowReplayInput, state: MemoryFlowInteractionState, io: KloMemoryFlowInteractiveIo): void {
|
||||
const view = buildMemoryFlowViewModel(input);
|
||||
io.stdout.write('\u001b[2J\u001b[H');
|
||||
io.stdout.write(renderMemoryFlowInteractive(view, state, { terminalWidth: io.stdout.columns }));
|
||||
}
|
||||
|
||||
export async function renderMemoryFlowInteractively(
|
||||
input: MemoryFlowReplayInput,
|
||||
io: KloMemoryFlowInteractiveIo,
|
||||
options: RenderMemoryFlowInteractiveOptions = {},
|
||||
): Promise<void> {
|
||||
const stdin = io.stdin;
|
||||
if (stdin?.isTTY !== true) {
|
||||
const view = buildMemoryFlowViewModel(input);
|
||||
io.stdout.write(
|
||||
renderMemoryFlowInteractive(view, createInitialMemoryFlowInteractionState(view), {
|
||||
terminalWidth: io.stdout.columns,
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const view = buildMemoryFlowViewModel(input);
|
||||
let state = createInitialMemoryFlowInteractionState(view);
|
||||
const previousRawMode = stdin.isRaw === true;
|
||||
|
||||
return new Promise((resolve) => {
|
||||
const cleanup = (): void => {
|
||||
removeKeypressListener(stdin, handleKeypress);
|
||||
stdin.setRawMode?.(previousRawMode);
|
||||
stdin.pause?.();
|
||||
};
|
||||
|
||||
const handleKeypress = (_chunk: string, key: KloMemoryFlowKey): void => {
|
||||
const command = memoryFlowCommandForKey(_chunk, state.search, key);
|
||||
if (!command) {
|
||||
return;
|
||||
}
|
||||
|
||||
state = reduceMemoryFlowInteractionState(state, command, view);
|
||||
repaint(input, state, io);
|
||||
|
||||
if (state.shouldQuit) {
|
||||
cleanup();
|
||||
resolve();
|
||||
}
|
||||
};
|
||||
|
||||
(options.prepareKeypressEvents ?? defaultPrepareKeypressEvents)(stdin);
|
||||
stdin.setRawMode?.(true);
|
||||
stdin.resume?.();
|
||||
stdin.on('keypress', handleKeypress);
|
||||
repaint(input, state, io);
|
||||
});
|
||||
}
|
||||
315
packages/cli/src/memory-flow-tui.test.tsx
Normal file
315
packages/cli/src/memory-flow-tui.test.tsx
Normal file
|
|
@ -0,0 +1,315 @@
|
|||
/* @jsxImportSource react */
|
||||
import type { MemoryFlowReplayInput } from '@klo/context/ingest';
|
||||
import { render as renderInkTest } from 'ink-testing-library';
|
||||
import React, { type ReactNode } from 'react';
|
||||
import { describe, expect, it, vi } from 'vitest';
|
||||
import {
|
||||
MemoryFlowTuiApp,
|
||||
memoryFlowCommandForInkInput,
|
||||
renderMemoryFlowTui,
|
||||
sanitizeMemoryFlowTuiError,
|
||||
startLiveMemoryFlowTui,
|
||||
type KloMemoryFlowTuiIo,
|
||||
type MemoryFlowInkInstance,
|
||||
type MemoryFlowInkRenderOptions,
|
||||
} from './memory-flow-tui.js';
|
||||
|
||||
function replayInput(): MemoryFlowReplayInput {
|
||||
return {
|
||||
runId: 'run-1', connectionId: 'warehouse', adapter: 'live-database',
|
||||
status: 'done', sourceDir: null, syncId: 'sync-1', reportId: 'report-1', reportPath: 'report-1', errors: [],
|
||||
plannedWorkUnits: [
|
||||
{ unitKey: 'orders', rawFiles: ['orders'], peerFileCount: 0, dependencyCount: 1 },
|
||||
{ unitKey: 'customers', rawFiles: ['customers'], peerFileCount: 1, dependencyCount: 0 },
|
||||
],
|
||||
details: {
|
||||
actions: [
|
||||
{ unitKey: 'orders', target: 'wiki', action: 'created', key: 'knowledge/orders.md', summary: 'order lifecycle', rawFiles: ['orders'], status: 'success' },
|
||||
{ unitKey: 'customers', target: 'sl', action: 'updated', key: 'orbit_demo.customers', summary: 'customer metrics', rawFiles: ['customers'], status: 'success' },
|
||||
],
|
||||
provenance: [{ rawPath: 'orders', artifactKind: 'wiki', artifactKey: 'knowledge/orders.md', actionType: 'wiki_written' }],
|
||||
transcripts: [{ unitKey: 'orders', path: '/tmp/t.jsonl', toolCallCount: 2, errorCount: 0, toolNames: ['read_raw_span', 'wiki_write'] }],
|
||||
},
|
||||
events: [
|
||||
{ type: 'source_acquired', adapter: 'live-database', trigger: 'manual_resync', fileCount: 2 },
|
||||
{ type: 'scope_detected', fingerprint: 'scope-1' },
|
||||
{ type: 'raw_snapshot_written', syncId: 'sync-1', rawFileCount: 2 },
|
||||
{ type: 'diff_computed', added: 1, modified: 1, deleted: 0, unchanged: 0 },
|
||||
{ type: 'chunks_planned', chunkCount: 2, workUnitCount: 2, evictionCount: 0 },
|
||||
{ type: 'work_unit_started', unitKey: 'orders', skills: ['knowledge_capture'], stepBudget: 40 },
|
||||
{ type: 'candidate_action', unitKey: 'orders', target: 'wiki', action: 'created', key: 'knowledge/orders.md' },
|
||||
{ type: 'work_unit_finished', unitKey: 'orders', status: 'success' },
|
||||
{ type: 'work_unit_started', unitKey: 'customers', skills: ['sl_capture'], stepBudget: 40 },
|
||||
{ type: 'candidate_action', unitKey: 'customers', target: 'sl', action: 'updated', key: 'orbit_demo.customers' },
|
||||
{ type: 'work_unit_finished', unitKey: 'customers', status: 'success' },
|
||||
{ type: 'reconciliation_finished', conflictCount: 0, fallbackCount: 0 },
|
||||
{ type: 'saved', commitSha: 'commit-one', wikiCount: 1, slCount: 1 },
|
||||
{ type: 'provenance_recorded', rowCount: 1 },
|
||||
{ type: 'report_created', runId: 'run-1', reportPath: 'report-1' },
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
function runningReplayInput(): MemoryFlowReplayInput {
|
||||
return { ...replayInput(), status: 'running', syncId: 'pending', reportId: undefined, reportPath: undefined, plannedWorkUnits: [], events: [{ type: 'source_acquired', adapter: 'live-database', trigger: 'manual_resync', fileCount: 1 }] };
|
||||
}
|
||||
|
||||
function packagedReplayInput(overrides: Partial<MemoryFlowReplayInput> = {}): MemoryFlowReplayInput {
|
||||
return {
|
||||
...replayInput(),
|
||||
connectionId: 'orbit_demo',
|
||||
metadata: {
|
||||
schemaVersion: 1,
|
||||
mode: 'seeded',
|
||||
origin: 'packaged',
|
||||
timing: 'prebuilt',
|
||||
capturedAt: null,
|
||||
sourceReportId: 'demo-seeded-report',
|
||||
sourceReportPath: 'reports/seeded-demo-report.json',
|
||||
fallbackReason: null,
|
||||
},
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function makeIo(): { io: KloMemoryFlowTuiIo; stderr: () => string } {
|
||||
let stderr = '';
|
||||
return { io: { stdin: { isTTY: true, setRawMode: vi.fn() }, stdout: { isTTY: true, columns: 120, write: vi.fn() }, stderr: { write(chunk: string) { stderr += chunk; } } }, stderr: () => stderr };
|
||||
}
|
||||
|
||||
function fakeInkInstance(): MemoryFlowInkInstance {
|
||||
return { rerender: vi.fn(), unmount: vi.fn(), waitUntilExit: vi.fn(async () => undefined), clear: vi.fn() };
|
||||
}
|
||||
|
||||
async function waitForInkInput(): Promise<void> { await new Promise((r) => setTimeout(r, 10)); }
|
||||
|
||||
function renderedAppProps(tree: ReactNode): Record<string, unknown> {
|
||||
expect(React.isValidElement(tree)).toBe(true);
|
||||
return (tree as React.ReactElement<Record<string, unknown>>).props;
|
||||
}
|
||||
|
||||
describe('memoryFlowCommandForInkInput', () => {
|
||||
it('maps input to commands', () => {
|
||||
expect(memoryFlowCommandForInkInput('q', {})).toBe('quit');
|
||||
expect(memoryFlowCommandForInkInput('c', { ctrl: true })).toBe('quit');
|
||||
expect(memoryFlowCommandForInkInput('x', {})).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('sanitizeMemoryFlowTuiError', () => {
|
||||
it('redacts credentials', () => {
|
||||
expect(sanitizeMemoryFlowTuiError(new Error('postgres://x?api_key=y password=z'))).toBe('[redacted-url] [redacted]');
|
||||
});
|
||||
});
|
||||
|
||||
describe('MemoryFlowTuiApp', () => {
|
||||
it('always shows the KLO logo', () => {
|
||||
const { lastFrame } = renderInkTest(<MemoryFlowTuiApp input={replayInput()} terminalWidth={120} onExit={vi.fn()} showBoot={false} />);
|
||||
expect(lastFrame()).toContain('█████╔╝');
|
||||
});
|
||||
|
||||
it('shows persistent HUD with source and status terminology', () => {
|
||||
const { lastFrame } = renderInkTest(<MemoryFlowTuiApp input={{ ...replayInput(), connectionId: 'warehouse' }} terminalWidth={120} onExit={vi.fn()} showBoot={false} />);
|
||||
const frame = lastFrame() ?? '';
|
||||
expect(frame).toContain('Database (warehouse)');
|
||||
expect(frame).toContain('2 tables');
|
||||
expect(frame).toContain('done');
|
||||
expect(frame).toContain('warehouse');
|
||||
expect(frame).toContain('╭');
|
||||
expect(frame).toContain('╰');
|
||||
});
|
||||
|
||||
it('hides the internal demo connection id before packaged replay source events are visible', () => {
|
||||
const { lastFrame } = renderInkTest(
|
||||
<MemoryFlowTuiApp
|
||||
input={packagedReplayInput({ status: 'running', events: [] })}
|
||||
terminalWidth={120}
|
||||
onExit={vi.fn()}
|
||||
showBoot={false}
|
||||
/>,
|
||||
);
|
||||
const frame = lastFrame() ?? '';
|
||||
expect(frame).toContain('Orbit Demo');
|
||||
expect(frame).not.toContain('orbit_demo');
|
||||
expect(frame).not.toContain('Database (orbit_demo)');
|
||||
});
|
||||
|
||||
it('keeps the packaged replay source label public while only one source event is visible', () => {
|
||||
const { lastFrame } = renderInkTest(
|
||||
<MemoryFlowTuiApp
|
||||
input={packagedReplayInput({
|
||||
status: 'running',
|
||||
events: [{ type: 'source_acquired', adapter: 'live-database', trigger: 'demo_seeded', fileCount: 8 }],
|
||||
})}
|
||||
terminalWidth={120}
|
||||
onExit={vi.fn()}
|
||||
showBoot={false}
|
||||
/>,
|
||||
);
|
||||
const frame = lastFrame() ?? '';
|
||||
expect(frame).toContain('Orbit Demo');
|
||||
expect(frame).not.toContain('orbit_demo');
|
||||
expect(frame).not.toContain('Database (orbit_demo)');
|
||||
});
|
||||
|
||||
it('shows a prepopulated data disclaimer for packaged demo replay cost estimates', () => {
|
||||
const { lastFrame } = renderInkTest(
|
||||
<MemoryFlowTuiApp
|
||||
input={packagedReplayInput()}
|
||||
terminalWidth={120}
|
||||
onExit={vi.fn()}
|
||||
showBoot={false}
|
||||
/>,
|
||||
);
|
||||
const frame = lastFrame() ?? '';
|
||||
expect(frame).toContain('$');
|
||||
expect(frame).toContain('Pre-run demo: $ shown is illustrative; no money is being spent now.');
|
||||
expect(frame).not.toContain('orbit_demo');
|
||||
});
|
||||
|
||||
it('does not show the prepopulated data disclaimer for captured full replay cost estimates', () => {
|
||||
const { lastFrame } = renderInkTest(
|
||||
<MemoryFlowTuiApp
|
||||
input={{
|
||||
...replayInput(),
|
||||
metadata: {
|
||||
schemaVersion: 1,
|
||||
mode: 'full',
|
||||
origin: 'captured',
|
||||
timing: 'captured',
|
||||
capturedAt: '2026-05-01T00:00:00.000Z',
|
||||
sourceReportId: 'report-full',
|
||||
sourceReportPath: 'reports/report-full.json',
|
||||
fallbackReason: null,
|
||||
},
|
||||
}}
|
||||
terminalWidth={120}
|
||||
onExit={vi.fn()}
|
||||
showBoot={false}
|
||||
/>,
|
||||
);
|
||||
expect(lastFrame()).not.toContain('Demo data is prepopulated');
|
||||
});
|
||||
|
||||
it('shows accumulated activity feed on completion', () => {
|
||||
const { lastFrame } = renderInkTest(<MemoryFlowTuiApp input={replayInput()} terminalWidth={120} onExit={vi.fn()} showBoot={false} />);
|
||||
const frame = lastFrame() ?? '';
|
||||
expect(frame).toContain('Connected — found 2 tables to ingest');
|
||||
expect(frame).toContain('Created so far:');
|
||||
expect(frame).toContain('order lifecycle');
|
||||
expect(frame).toContain('customer metrics');
|
||||
expect(frame).toContain('KLO finished ingesting your data');
|
||||
expect(frame).toContain('klo sl list');
|
||||
expect(frame).toContain('klo wiki list');
|
||||
expect(frame).toContain('klo serve --mcp stdio --user-id local');
|
||||
expect(frame).not.toContain(['klo', 'ask'].join(' '));
|
||||
expect(frame).not.toContain(['klo', 'mcp'].join(' '));
|
||||
});
|
||||
|
||||
it('handles quit while running', async () => {
|
||||
const onExit = vi.fn();
|
||||
const { stdin } = renderInkTest(<MemoryFlowTuiApp input={runningReplayInput()} terminalWidth={120} onExit={onExit} showBoot={false} />);
|
||||
stdin.write('q');
|
||||
await waitForInkInput();
|
||||
expect(onExit).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('shows active work unit with progress', () => {
|
||||
const running: MemoryFlowReplayInput = {
|
||||
...runningReplayInput(),
|
||||
events: [
|
||||
{ type: 'source_acquired', adapter: 'live-database', trigger: 'manual_resync', fileCount: 1 },
|
||||
{ type: 'diff_computed', added: 1, modified: 0, deleted: 0, unchanged: 0 },
|
||||
{ type: 'chunks_planned', chunkCount: 1, workUnitCount: 1, evictionCount: 0 },
|
||||
{ type: 'work_unit_started', unitKey: 'orders', skills: ['knowledge_capture'], stepBudget: 40 },
|
||||
],
|
||||
plannedWorkUnits: [{ unitKey: 'orders', rawFiles: ['orders'], peerFileCount: 0, dependencyCount: 1 }],
|
||||
};
|
||||
const { lastFrame } = renderInkTest(<MemoryFlowTuiApp input={running} terminalWidth={120} onExit={vi.fn()} showBoot={false} />);
|
||||
const frame = lastFrame() ?? '';
|
||||
expect(frame).toContain('Ingesting — 0/1 business area done');
|
||||
expect(frame).toContain('Reading table schemas, understanding relationships, creating query definitions');
|
||||
expect(frame).toContain('█████╔╝');
|
||||
});
|
||||
|
||||
it('describes multi-source ingestion as building the context layer', () => {
|
||||
const running: MemoryFlowReplayInput = {
|
||||
...runningReplayInput(),
|
||||
adapter: 'multi-source',
|
||||
events: [
|
||||
{ type: 'source_acquired', adapter: 'live-database', trigger: 'manual_resync', fileCount: 8 },
|
||||
{ type: 'source_acquired', adapter: 'dbt-descriptions', trigger: 'manual_resync', fileCount: 3 },
|
||||
{ type: 'diff_computed', added: 11, modified: 0, deleted: 0, unchanged: 0 },
|
||||
{ type: 'chunks_planned', chunkCount: 1, workUnitCount: 1, evictionCount: 0 },
|
||||
{ type: 'work_unit_started', unitKey: 'orders', skills: ['knowledge_capture'], stepBudget: 40 },
|
||||
],
|
||||
plannedWorkUnits: [{ unitKey: 'orders', rawFiles: ['orders'], peerFileCount: 0, dependencyCount: 1 }],
|
||||
};
|
||||
|
||||
const { lastFrame } = renderInkTest(<MemoryFlowTuiApp input={running} terminalWidth={120} onExit={vi.fn()} showBoot={false} />);
|
||||
const frame = lastFrame() ?? '';
|
||||
expect(frame).toContain('Ingesting warehouse, dbt, BI, and docs into a unified context layer');
|
||||
expect(frame).not.toContain('unified semantic layer');
|
||||
});
|
||||
|
||||
it('hides completion while running', () => {
|
||||
const { lastFrame } = renderInkTest(<MemoryFlowTuiApp input={runningReplayInput()} terminalWidth={120} onExit={vi.fn()} showBoot={false} />);
|
||||
expect(lastFrame()).not.toContain('KLO finished ingesting');
|
||||
});
|
||||
});
|
||||
|
||||
describe('startLiveMemoryFlowTui', () => {
|
||||
it('starts and updates', async () => {
|
||||
const { io } = makeIo();
|
||||
const instance = fakeInkInstance();
|
||||
const live = await startLiveMemoryFlowTui(runningReplayInput(), io, { renderInk: () => instance });
|
||||
expect(live).not.toBeNull();
|
||||
live?.update(replayInput());
|
||||
expect(instance.rerender).toHaveBeenCalledTimes(1);
|
||||
live?.close();
|
||||
expect(instance.unmount).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('redacts errors', async () => {
|
||||
const { io, stderr } = makeIo();
|
||||
await expect(startLiveMemoryFlowTui(runningReplayInput(), io, { renderInk: () => { throw new Error('postgres://x?token=y'); } })).resolves.toBeNull();
|
||||
expect(stderr()).toContain('[redacted-url]');
|
||||
});
|
||||
});
|
||||
|
||||
describe('renderMemoryFlowTui', () => {
|
||||
it('renders and returns true', async () => {
|
||||
const { io } = makeIo();
|
||||
const instance = fakeInkInstance();
|
||||
await expect(renderMemoryFlowTui(replayInput(), io, { renderInk: () => instance })).resolves.toBe(true);
|
||||
});
|
||||
|
||||
it('scales event timing with the speed multiplier while keeping animations normal speed', async () => {
|
||||
const { io } = makeIo();
|
||||
const instance = fakeInkInstance();
|
||||
let renderedTree: ReactNode = null;
|
||||
|
||||
await expect(
|
||||
renderMemoryFlowTui(replayInput(), io, {
|
||||
speedMultiplier: 0.125,
|
||||
renderInk: (tree) => {
|
||||
renderedTree = tree;
|
||||
return instance;
|
||||
},
|
||||
}),
|
||||
).resolves.toBe(true);
|
||||
|
||||
expect(renderedAppProps(renderedTree)).toMatchObject({
|
||||
paceMsPerEvent: 1440,
|
||||
frameMs: 140,
|
||||
completionFrameMs: 80,
|
||||
completionHoldMs: 1000,
|
||||
});
|
||||
});
|
||||
|
||||
it('redacts errors', async () => {
|
||||
const { io, stderr } = makeIo();
|
||||
await expect(renderMemoryFlowTui(replayInput(), io, { renderInk: () => { throw new Error('postgres://x?token=y'); } })).resolves.toBe(false);
|
||||
expect(stderr()).toContain('[redacted-url]');
|
||||
});
|
||||
});
|
||||
552
packages/cli/src/memory-flow-tui.tsx
Normal file
552
packages/cli/src/memory-flow-tui.tsx
Normal file
|
|
@ -0,0 +1,552 @@
|
|||
/* @jsxImportSource react */
|
||||
import {
|
||||
buildMemoryFlowViewModel,
|
||||
buildMemoryFlowVisualModel,
|
||||
createInitialMemoryFlowInteractionState,
|
||||
findMemoryFlowSearchMatches,
|
||||
type MemoryFlowColumnId,
|
||||
type MemoryFlowInteractionCommand,
|
||||
type MemoryFlowInteractionState,
|
||||
type MemoryFlowReplayInput,
|
||||
type MemoryFlowViewModel,
|
||||
reduceMemoryFlowInteractionState,
|
||||
selectedMemoryFlowColumn,
|
||||
selectedMemoryFlowDetails,
|
||||
} from '@klo/context/ingest';
|
||||
import { Box, Text, render as renderInkRuntime, useApp, useInput } from 'ink';
|
||||
import React, { type ReactNode, useEffect, useMemo, useRef, useState } from 'react';
|
||||
import { buildDemoMetrics } from './demo-metrics.js';
|
||||
import {
|
||||
ActivityFeed,
|
||||
Hud,
|
||||
Logo,
|
||||
} from './memory-flow-hud.js';
|
||||
import { profileMark } from './startup-profile.js';
|
||||
|
||||
profileMark('module:memory-flow-tui');
|
||||
|
||||
const COLOR_THEME = {
|
||||
text: 'white',
|
||||
muted: 'gray',
|
||||
active: 'cyan',
|
||||
complete: 'green',
|
||||
warning: 'yellow',
|
||||
failed: 'red',
|
||||
border: 'gray',
|
||||
} as const;
|
||||
|
||||
const NO_COLOR_THEME = {
|
||||
text: 'white',
|
||||
muted: 'white',
|
||||
active: 'white',
|
||||
complete: 'white',
|
||||
warning: 'white',
|
||||
failed: 'white',
|
||||
border: 'white',
|
||||
} as const;
|
||||
|
||||
type MemoryFlowTuiTheme = Record<keyof typeof COLOR_THEME, string>;
|
||||
|
||||
const STAGE_LABELS = {
|
||||
source: 'CONNECT',
|
||||
chunks: 'SNAPSHOT',
|
||||
workUnits: 'PLAN',
|
||||
actions: 'ANALYZE',
|
||||
gates: 'VALIDATE',
|
||||
saved: 'MEMORY',
|
||||
} satisfies Record<MemoryFlowColumnId, string>;
|
||||
|
||||
export interface KloMemoryFlowTuiIo {
|
||||
stdin?: { isTTY?: boolean; setRawMode?(value: boolean): void };
|
||||
stdout: { isTTY?: boolean; columns?: number; write(chunk: string): void };
|
||||
stderr: { write(chunk: string): void };
|
||||
}
|
||||
|
||||
export interface MemoryFlowTuiLiveSession {
|
||||
update(input: MemoryFlowReplayInput): void;
|
||||
close(): void;
|
||||
isClosed(): boolean;
|
||||
}
|
||||
|
||||
export interface MemoryFlowInkInstance {
|
||||
rerender(tree: ReactNode): void;
|
||||
unmount(): void;
|
||||
waitUntilExit(): Promise<void>;
|
||||
clear?(): void;
|
||||
}
|
||||
|
||||
export interface MemoryFlowInkRenderOptions {
|
||||
stdin?: KloMemoryFlowTuiIo['stdin'];
|
||||
stdout: KloMemoryFlowTuiIo['stdout'];
|
||||
stderr: KloMemoryFlowTuiIo['stderr'];
|
||||
exitOnCtrlC: boolean;
|
||||
patchConsole: boolean;
|
||||
maxFps: number;
|
||||
alternateScreen: boolean;
|
||||
}
|
||||
|
||||
interface RenderMemoryFlowTuiOptions {
|
||||
renderInk?: (tree: ReactNode, options: MemoryFlowInkRenderOptions) => MemoryFlowInkInstance;
|
||||
paceEvents?: boolean;
|
||||
paceMsPerEvent?: number;
|
||||
speedMultiplier?: number;
|
||||
}
|
||||
|
||||
interface StartLiveMemoryFlowTuiOptions {
|
||||
renderInk?: (tree: ReactNode, options: MemoryFlowInkRenderOptions) => MemoryFlowInkInstance;
|
||||
}
|
||||
|
||||
interface RenderTreeOptions {
|
||||
paceEvents?: boolean;
|
||||
paceMsPerEvent?: number;
|
||||
frameMs?: number;
|
||||
completionFrameMs?: number;
|
||||
completionHoldMs?: number;
|
||||
}
|
||||
|
||||
interface MemoryFlowTuiTiming {
|
||||
paceMsPerEvent: number;
|
||||
frameMs: number;
|
||||
completionFrameMs: number;
|
||||
completionHoldMs: number;
|
||||
}
|
||||
|
||||
const DEFAULT_TUI_TIMING = {
|
||||
paceMsPerEvent: 180,
|
||||
frameMs: 140,
|
||||
completionFrameMs: 80,
|
||||
completionHoldMs: 1000,
|
||||
} satisfies MemoryFlowTuiTiming;
|
||||
|
||||
interface InkKey {
|
||||
leftArrow?: boolean;
|
||||
rightArrow?: boolean;
|
||||
upArrow?: boolean;
|
||||
downArrow?: boolean;
|
||||
return?: boolean;
|
||||
escape?: boolean;
|
||||
ctrl?: boolean;
|
||||
shift?: boolean;
|
||||
tab?: boolean;
|
||||
backspace?: boolean;
|
||||
delete?: boolean;
|
||||
}
|
||||
|
||||
interface MemoryFlowTuiAppProps {
|
||||
input: MemoryFlowReplayInput;
|
||||
terminalWidth?: number;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
onExit(): void;
|
||||
paceEvents?: boolean;
|
||||
paceMsPerEvent?: number;
|
||||
frameMs?: number;
|
||||
completionFrameMs?: number;
|
||||
completionHoldMs?: number;
|
||||
showBoot?: boolean;
|
||||
}
|
||||
|
||||
function resolveMemoryFlowTuiTheme(env: NodeJS.ProcessEnv = process.env): MemoryFlowTuiTheme {
|
||||
if (env.NO_COLOR || env.TERM === 'dumb') {
|
||||
return NO_COLOR_THEME;
|
||||
}
|
||||
return COLOR_THEME;
|
||||
}
|
||||
|
||||
export function sanitizeMemoryFlowTuiError(error: unknown): string {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
return message
|
||||
.replace(/[a-z][a-z0-9+.-]*:\/\/[^\s]+/gi, '[redacted-url]')
|
||||
.replace(/\b(api[_-]?key|password|token|secret)=\S+/gi, '[redacted]');
|
||||
}
|
||||
|
||||
export function memoryFlowCommandForInkInput(
|
||||
input: string,
|
||||
key: InkKey,
|
||||
search: MemoryFlowInteractionState['search'] = { editing: false, query: '', matchIndex: 0 },
|
||||
): MemoryFlowInteractionCommand | null {
|
||||
if (search.editing) {
|
||||
if (key.escape) return 'search-clear';
|
||||
if (key.return) return 'search-submit';
|
||||
if (key.backspace || key.delete) return 'search-backspace';
|
||||
if (key.downArrow || (key.tab && !key.shift)) return 'search-next';
|
||||
if (key.upArrow || (key.tab && key.shift)) return 'search-previous';
|
||||
if (input.length === 1 && input >= ' ' && input !== '') {
|
||||
return { type: 'search-input', value: input };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
if (key.ctrl === true && input === 'c') return 'quit';
|
||||
if (input === '/') return 'search-start';
|
||||
if (search.query && input === 'n') return 'search-next';
|
||||
if (search.query && input === 'N') return 'search-previous';
|
||||
if (input === '[D') return 'left';
|
||||
if (input === '[C') return 'right';
|
||||
if (input === '[A') return 'up';
|
||||
if (input === '[B') return 'down';
|
||||
if (key.leftArrow) return 'left';
|
||||
if (key.rightArrow) return 'right';
|
||||
if (key.upArrow) return 'up';
|
||||
if (key.downArrow) return 'down';
|
||||
if (key.return) return 'enter';
|
||||
if (key.tab) return 'tab';
|
||||
if (input === 'f') return 'filter';
|
||||
if (input === 'p') return 'provenance';
|
||||
if (input === 't') return 'transcript';
|
||||
if (input === 'q' || key.escape) return 'quit';
|
||||
return null;
|
||||
}
|
||||
|
||||
function stageLabel(columnId: MemoryFlowColumnId): string {
|
||||
return STAGE_LABELS[columnId];
|
||||
}
|
||||
|
||||
function statusLabel(status: string): 'OK' | 'RUN' | 'WARN' | 'FAIL' | 'WAIT' {
|
||||
if (status === 'complete') return 'OK';
|
||||
if (status === 'active') return 'RUN';
|
||||
if (status === 'warning') return 'WARN';
|
||||
if (status === 'failed') return 'FAIL';
|
||||
return 'WAIT';
|
||||
}
|
||||
|
||||
function filterLabel(filter: MemoryFlowInteractionState['filter']): string {
|
||||
return filter === 'failed_or_flagged' ? 'issues' : 'all';
|
||||
}
|
||||
|
||||
function searchStatusLine(view: MemoryFlowViewModel, state: MemoryFlowInteractionState): string | null {
|
||||
if (!state.search.editing && state.search.query.length === 0) {
|
||||
return null;
|
||||
}
|
||||
const matches = findMemoryFlowSearchMatches(view, state.search.query);
|
||||
const status = state.search.editing ? 'editing' : 'locked';
|
||||
const position = matches.length === 0 ? '0/0' : `${state.search.matchIndex + 1}/${matches.length}`;
|
||||
return `Search: ${state.search.query || '/'} (${position} matches, ${status})`;
|
||||
}
|
||||
|
||||
function humanizeDemoText(value: string): string {
|
||||
return value
|
||||
.replace(/\bWORKUNITS\b/g, 'PLAN')
|
||||
.replace(/\bWorkUnit\b/g, 'Table review')
|
||||
.replace(/\bwork units\b/gi, 'table reviews')
|
||||
.replace(/\bwork-unit\b/gi, 'table-review')
|
||||
.replace(/\bWUs\b/g, 'tables')
|
||||
.replace(/\bchunks\b/gi, 'table groups')
|
||||
.replace(/\bcandidates\b/gi, 'drafts')
|
||||
.replace(/\bcandidate\b/gi, 'draft')
|
||||
.replace(/\braw files\b/gi, 'database files')
|
||||
.replace(/\braw file\b/gi, 'database file')
|
||||
.replace(/\bSL\b/g, 'context layer');
|
||||
}
|
||||
|
||||
function DetailsPane(props: {
|
||||
view: MemoryFlowViewModel;
|
||||
state: MemoryFlowInteractionState;
|
||||
theme: MemoryFlowTuiTheme;
|
||||
}): ReactNode {
|
||||
const column = selectedMemoryFlowColumn(props.view, props.state);
|
||||
const details = selectedMemoryFlowDetails(props.view, props.state).map(humanizeDemoText).slice(0, 8);
|
||||
const rawFiles = Array.from(
|
||||
new Set([
|
||||
...props.view.details.actions.flatMap((action) => action.rawFiles),
|
||||
...props.view.details.provenance.map((row) => row.rawPath),
|
||||
]),
|
||||
).slice(0, 4);
|
||||
const searchLine = searchStatusLine(props.view, props.state);
|
||||
|
||||
return (
|
||||
<Box flexDirection="column">
|
||||
<Text color={props.theme.active}>
|
||||
Details / focus: {stageLabel(column.id)} Pane: {props.state.pane} Filter: {filterLabel(props.state.filter)}
|
||||
</Text>
|
||||
{searchLine && <Text color={props.theme.active}>{searchLine}</Text>}
|
||||
{details.map((detail, index) => (
|
||||
<Text key={`${index}-${detail}`} color={props.theme.text}>
|
||||
- {detail}
|
||||
</Text>
|
||||
))}
|
||||
{rawFiles.map((rawFile) => (
|
||||
<Text key={rawFile} color={props.theme.muted}>
|
||||
- {rawFile}
|
||||
</Text>
|
||||
))}
|
||||
{props.view.completionLine && <Text color={props.theme.complete}>{humanizeDemoText(props.view.completionLine)}</Text>}
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
|
||||
function TrustIssues(props: { view: MemoryFlowViewModel; theme: MemoryFlowTuiTheme }): ReactNode {
|
||||
if (props.view.trustIssues.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return (
|
||||
<Box flexDirection="column" marginBottom={1}>
|
||||
<Text color={props.theme.warning}>Validation notes</Text>
|
||||
{props.view.trustIssues.slice(0, 4).map((issue) => (
|
||||
<Text
|
||||
key={`${issue.severity}-${issue.title}`}
|
||||
color={issue.severity === 'failed' ? props.theme.failed : props.theme.warning}
|
||||
>
|
||||
{issue.severity === 'failed' ? 'FAILED' : 'WARNING'} {humanizeDemoText(issue.title)}:{' '}
|
||||
{humanizeDemoText(issue.detail)}
|
||||
</Text>
|
||||
))}
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
|
||||
export function MemoryFlowTuiApp(props: MemoryFlowTuiAppProps): ReactNode {
|
||||
const app = useApp();
|
||||
const totalEvents = props.input.events.length;
|
||||
const paceEnabled = props.paceEvents === true && totalEvents > 0;
|
||||
const [pacedCount, setPacedCount] = useState<number>(paceEnabled ? 0 : totalEvents);
|
||||
|
||||
const pacedInput = useMemo<MemoryFlowReplayInput>(() => {
|
||||
if (!paceEnabled || pacedCount >= totalEvents) {
|
||||
return props.input;
|
||||
}
|
||||
return {
|
||||
...props.input,
|
||||
status: 'running',
|
||||
events: props.input.events.slice(0, pacedCount),
|
||||
};
|
||||
}, [paceEnabled, pacedCount, totalEvents, props.input]);
|
||||
|
||||
const pacedNow = useMemo<(() => number) | undefined>(() => {
|
||||
if (!paceEnabled) return undefined;
|
||||
const firstEvent = props.input.events[0];
|
||||
if (!firstEvent?.emittedAt) return undefined;
|
||||
const firstEventMs = Date.parse(firstEvent.emittedAt);
|
||||
if (!Number.isFinite(firstEventMs)) return undefined;
|
||||
const stride = props.paceMsPerEvent ?? DEFAULT_TUI_TIMING.paceMsPerEvent;
|
||||
return () => firstEventMs + pacedCount * stride;
|
||||
}, [paceEnabled, pacedCount, props.input.events, props.paceMsPerEvent]);
|
||||
|
||||
const view = useMemo(() => buildMemoryFlowViewModel(pacedInput), [pacedInput]);
|
||||
const [state, setState] = useState<MemoryFlowInteractionState>(() => createInitialMemoryFlowInteractionState(view));
|
||||
const [frame, setFrame] = useState(0);
|
||||
const [thoughtFrame, setThoughtFrame] = useState(0);
|
||||
const [completionFrame, setCompletionFrame] = useState(0);
|
||||
const [holdComplete, setHoldComplete] = useState(false);
|
||||
const [userHasNavigated, setUserHasNavigated] = useState(false);
|
||||
const lastEventCountRef = useRef(pacedInput.events.length);
|
||||
const lastStatusRef = useRef(pacedInput.status);
|
||||
const exitHandled = useRef(false);
|
||||
const theme = resolveMemoryFlowTuiTheme(props.env);
|
||||
|
||||
useEffect(() => {
|
||||
if (!state.shouldQuit || exitHandled.current) {
|
||||
return;
|
||||
}
|
||||
exitHandled.current = true;
|
||||
props.onExit();
|
||||
app.exit();
|
||||
}, [app, props, state.shouldQuit]);
|
||||
|
||||
useEffect(() => {
|
||||
const timer = setInterval(() => {
|
||||
setFrame((current) => current + 1);
|
||||
setThoughtFrame((current) => current + 1);
|
||||
}, props.frameMs ?? DEFAULT_TUI_TIMING.frameMs);
|
||||
return () => clearInterval(timer);
|
||||
}, [props.frameMs]);
|
||||
|
||||
useEffect(() => {
|
||||
if (lastEventCountRef.current !== pacedInput.events.length) {
|
||||
lastEventCountRef.current = pacedInput.events.length;
|
||||
setThoughtFrame(0);
|
||||
}
|
||||
}, [pacedInput.events.length]);
|
||||
|
||||
useEffect(() => {
|
||||
if (lastStatusRef.current !== pacedInput.status) {
|
||||
lastStatusRef.current = pacedInput.status;
|
||||
if (pacedInput.status === 'done' || pacedInput.status === 'error') {
|
||||
setCompletionFrame(0);
|
||||
}
|
||||
}
|
||||
}, [pacedInput.status]);
|
||||
|
||||
useEffect(() => {
|
||||
if (pacedInput.status !== 'done' && pacedInput.status !== 'error') return;
|
||||
if (completionFrame >= 12) return;
|
||||
const timer = setInterval(
|
||||
() => setCompletionFrame((current) => Math.min(12, current + 1)),
|
||||
props.completionFrameMs ?? DEFAULT_TUI_TIMING.completionFrameMs,
|
||||
);
|
||||
return () => clearInterval(timer);
|
||||
}, [pacedInput.status, completionFrame, props.completionFrameMs]);
|
||||
|
||||
useEffect(() => {
|
||||
if (completionFrame < 12) {
|
||||
setHoldComplete(false);
|
||||
return;
|
||||
}
|
||||
const timer = setTimeout(
|
||||
() => setHoldComplete(true),
|
||||
props.completionHoldMs ?? DEFAULT_TUI_TIMING.completionHoldMs,
|
||||
);
|
||||
return () => clearTimeout(timer);
|
||||
}, [completionFrame, props.completionHoldMs]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!paceEnabled || pacedCount >= totalEvents) {
|
||||
return;
|
||||
}
|
||||
const interval = props.paceMsPerEvent ?? DEFAULT_TUI_TIMING.paceMsPerEvent;
|
||||
const timer = setInterval(() => {
|
||||
setPacedCount((current) => Math.min(totalEvents, current + 1));
|
||||
}, interval);
|
||||
return () => clearInterval(timer);
|
||||
}, [paceEnabled, pacedCount, totalEvents, props.paceMsPerEvent]);
|
||||
|
||||
useInput((input, key) => {
|
||||
const command = memoryFlowCommandForInkInput(input, key, state.search);
|
||||
if (!command) return;
|
||||
if (command === 'quit' && isComplete && !holdComplete) return;
|
||||
if (command !== 'quit') setUserHasNavigated(true);
|
||||
setState((current) => reduceMemoryFlowInteractionState(current, command, view));
|
||||
});
|
||||
|
||||
const isComplete = pacedInput.status === 'done' || pacedInput.status === 'error';
|
||||
const completionMetrics = useMemo(
|
||||
() => buildDemoMetrics(pacedInput, pacedNow ? { now: pacedNow } : {}),
|
||||
[pacedInput, pacedNow],
|
||||
);
|
||||
|
||||
const termWidth = props.terminalWidth ?? 80;
|
||||
|
||||
return (
|
||||
<Box flexDirection="column" paddingX={1}>
|
||||
<Logo theme={theme} done={isComplete} />
|
||||
<Hud input={pacedInput} theme={theme} frame={frame} width={termWidth} now={pacedNow} />
|
||||
<ActivityFeed input={pacedInput} theme={theme} frame={frame} width={termWidth} completionFrame={completionFrame} showCompletion={isComplete} holdComplete={holdComplete} />
|
||||
<TrustIssues view={view} theme={theme} />
|
||||
{userHasNavigated && <DetailsPane view={view} state={state} theme={theme} />}
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
|
||||
function renderTree(
|
||||
input: MemoryFlowReplayInput,
|
||||
io: KloMemoryFlowTuiIo,
|
||||
onExit: () => void,
|
||||
options: RenderTreeOptions = {},
|
||||
): ReactNode {
|
||||
return (
|
||||
<MemoryFlowTuiApp
|
||||
input={input}
|
||||
terminalWidth={io.stdout.columns ?? process.stdout.columns}
|
||||
onExit={onExit}
|
||||
paceEvents={options.paceEvents}
|
||||
paceMsPerEvent={options.paceMsPerEvent}
|
||||
frameMs={options.frameMs}
|
||||
completionFrameMs={options.completionFrameMs}
|
||||
completionHoldMs={options.completionHoldMs}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
function renderInk(tree: ReactNode, options: MemoryFlowInkRenderOptions): MemoryFlowInkInstance {
|
||||
return renderInkRuntime(tree, {
|
||||
stdin: options.stdin as NodeJS.ReadStream | undefined,
|
||||
stdout: options.stdout as NodeJS.WriteStream,
|
||||
stderr: options.stderr as NodeJS.WriteStream,
|
||||
exitOnCtrlC: options.exitOnCtrlC,
|
||||
patchConsole: options.patchConsole,
|
||||
maxFps: options.maxFps,
|
||||
alternateScreen: options.alternateScreen,
|
||||
}) as MemoryFlowInkInstance;
|
||||
}
|
||||
|
||||
function renderOptions(io: KloMemoryFlowTuiIo): MemoryFlowInkRenderOptions {
|
||||
return {
|
||||
stdin: io.stdin,
|
||||
stdout: io.stdout,
|
||||
stderr: io.stderr,
|
||||
exitOnCtrlC: false,
|
||||
patchConsole: false,
|
||||
maxFps: 30,
|
||||
alternateScreen: true,
|
||||
};
|
||||
}
|
||||
|
||||
function scaleTiming(ms: number, speedMultiplier: number): number {
|
||||
return Math.max(20, Math.round(ms / speedMultiplier));
|
||||
}
|
||||
|
||||
function resolveTiming(options: RenderMemoryFlowTuiOptions): MemoryFlowTuiTiming {
|
||||
const speedMultiplier =
|
||||
typeof options.speedMultiplier === 'number' && options.speedMultiplier > 0 ? options.speedMultiplier : 1;
|
||||
return {
|
||||
paceMsPerEvent:
|
||||
typeof options.paceMsPerEvent === 'number' && options.paceMsPerEvent > 0
|
||||
? options.paceMsPerEvent
|
||||
: scaleTiming(DEFAULT_TUI_TIMING.paceMsPerEvent, speedMultiplier),
|
||||
frameMs: DEFAULT_TUI_TIMING.frameMs,
|
||||
completionFrameMs: DEFAULT_TUI_TIMING.completionFrameMs,
|
||||
completionHoldMs: DEFAULT_TUI_TIMING.completionHoldMs,
|
||||
};
|
||||
}
|
||||
|
||||
export async function renderMemoryFlowTui(
|
||||
input: MemoryFlowReplayInput,
|
||||
io: KloMemoryFlowTuiIo,
|
||||
options: RenderMemoryFlowTuiOptions = {},
|
||||
): Promise<boolean> {
|
||||
let instance: MemoryFlowInkInstance | null = null;
|
||||
const paceEvents = options.paceEvents !== false;
|
||||
const timing = resolveTiming(options);
|
||||
try {
|
||||
const onExit = (): void => {
|
||||
instance?.unmount();
|
||||
};
|
||||
instance = (options.renderInk ?? renderInk)(
|
||||
renderTree(input, io, onExit, { paceEvents, ...timing }),
|
||||
renderOptions(io),
|
||||
);
|
||||
await instance.waitUntilExit();
|
||||
instance.unmount();
|
||||
return true;
|
||||
} catch (error) {
|
||||
io.stderr.write(`TUI visualization unavailable: ${sanitizeMemoryFlowTuiError(error)}; using text renderer.\n`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export async function startLiveMemoryFlowTui(
|
||||
input: MemoryFlowReplayInput,
|
||||
io: KloMemoryFlowTuiIo,
|
||||
options: StartLiveMemoryFlowTuiOptions = {},
|
||||
): Promise<MemoryFlowTuiLiveSession | null> {
|
||||
let instance: MemoryFlowInkInstance | null = null;
|
||||
let closed = false;
|
||||
|
||||
const close = (): void => {
|
||||
if (closed) {
|
||||
return;
|
||||
}
|
||||
closed = true;
|
||||
instance?.unmount();
|
||||
};
|
||||
|
||||
try {
|
||||
instance = (options.renderInk ?? renderInk)(renderTree(input, io, close), renderOptions(io));
|
||||
|
||||
return {
|
||||
update(nextInput: MemoryFlowReplayInput): void {
|
||||
if (closed) {
|
||||
return;
|
||||
}
|
||||
instance?.rerender(renderTree(nextInput, io, close));
|
||||
},
|
||||
close,
|
||||
isClosed(): boolean {
|
||||
return closed;
|
||||
},
|
||||
};
|
||||
} catch (error) {
|
||||
io.stderr.write(`TUI visualization unavailable: ${sanitizeMemoryFlowTuiError(error)}; using text renderer.\n`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
129
packages/cli/src/next-steps.test.ts
Normal file
129
packages/cli/src/next-steps.test.ts
Normal file
|
|
@ -0,0 +1,129 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
KLO_CONTEXT_BUILD_COMMANDS,
|
||||
KLO_NEXT_STEP_COMMANDS,
|
||||
formatNextStepLines,
|
||||
formatSetupNextStepLines,
|
||||
} from './next-steps.js';
|
||||
|
||||
const command = (...parts: string[]) => parts.join(' ');
|
||||
|
||||
describe('KLO demo next steps', () => {
|
||||
it('uses supported context-build commands before agent usage', () => {
|
||||
expect(KLO_CONTEXT_BUILD_COMMANDS).toEqual([
|
||||
{
|
||||
command: 'klo setup context build',
|
||||
description: 'Build agent-ready context from configured primary and context sources',
|
||||
},
|
||||
{
|
||||
command: 'klo status',
|
||||
description: 'Check setup and context readiness',
|
||||
},
|
||||
{
|
||||
command: 'klo setup context status',
|
||||
description: 'Check the setup-managed context build state',
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it('uses supported final public commands', () => {
|
||||
expect(KLO_NEXT_STEP_COMMANDS).toEqual([
|
||||
{
|
||||
command: 'klo agent context --json',
|
||||
description: 'Verify the project context your agent can read',
|
||||
},
|
||||
{
|
||||
command: 'klo agent tools --json',
|
||||
description: 'List direct CLI tools available to agents',
|
||||
},
|
||||
{
|
||||
command: 'klo sl list',
|
||||
description: 'Inspect generated semantic-layer sources',
|
||||
},
|
||||
{
|
||||
command: 'klo wiki list',
|
||||
description: 'Inspect generated wiki pages',
|
||||
},
|
||||
{
|
||||
command: 'klo serve --mcp stdio --user-id local',
|
||||
description: 'Optional MCP server route for clients that require MCP',
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it('prefers the direct CLI route before MCP serving', () => {
|
||||
const commands = KLO_NEXT_STEP_COMMANDS.map((step) => step.command);
|
||||
|
||||
expect(commands.indexOf('klo agent context --json')).toBeLessThan(
|
||||
commands.indexOf('klo serve --mcp stdio --user-id local'),
|
||||
);
|
||||
expect(commands.indexOf('klo agent tools --json')).toBeLessThan(
|
||||
commands.indexOf('klo serve --mcp stdio --user-id local'),
|
||||
);
|
||||
});
|
||||
|
||||
it('explains what the next-step commands are for', () => {
|
||||
const rendered = formatNextStepLines().join('\n');
|
||||
|
||||
expect(rendered).toContain('KLO context is ready for agents.');
|
||||
expect(rendered).toContain('Preferred route: CLI + Skills');
|
||||
expect(rendered).toContain('no MCP server is required');
|
||||
expect(rendered).toContain('Direct CLI checks:');
|
||||
expect(rendered).toContain('Optional MCP:');
|
||||
expect(rendered).not.toContain('Ask your agent to use KLO');
|
||||
});
|
||||
|
||||
it('does not advertise removed Commander migration commands', () => {
|
||||
const rendered = formatNextStepLines().join('\n');
|
||||
|
||||
expect(rendered).toContain('klo agent tools --json');
|
||||
expect(rendered).toContain('klo agent context --json');
|
||||
expect(rendered).toContain('klo sl list');
|
||||
expect(rendered).toContain('klo wiki list');
|
||||
expect(rendered).toContain('klo serve --mcp stdio --user-id local');
|
||||
|
||||
for (const removed of [
|
||||
command('klo', 'ask'),
|
||||
command('klo', 'mcp'),
|
||||
command('klo', 'connect'),
|
||||
command('klo', 'knowledge'),
|
||||
command('dev', 'model'),
|
||||
command('dev', 'knowledge'),
|
||||
command('klo', 'ingest', 'run'),
|
||||
command('klo', 'ingest', 'replay'),
|
||||
]) {
|
||||
expect(rendered).not.toContain(removed);
|
||||
}
|
||||
});
|
||||
|
||||
it('keeps setup next steps focused on building context when the build is not ready', () => {
|
||||
const rendered = formatSetupNextStepLines({
|
||||
setupReady: true,
|
||||
hasContextTargets: true,
|
||||
contextReady: false,
|
||||
agentIntegrationReady: true,
|
||||
}).join('\n');
|
||||
|
||||
expect(rendered).toContain('Build KLO context next.');
|
||||
expect(rendered).toContain('primary-source scans and context-source ingests');
|
||||
expect(rendered).toContain('klo setup context build');
|
||||
expect(rendered).toContain('klo status');
|
||||
expect(rendered).toContain('klo setup context status');
|
||||
expect(rendered).not.toContain('klo agent context --json');
|
||||
expect(rendered).not.toContain('klo serve --mcp');
|
||||
});
|
||||
|
||||
it('shows agent commands only after setup and context build are ready', () => {
|
||||
const rendered = formatSetupNextStepLines({
|
||||
setupReady: true,
|
||||
hasContextTargets: true,
|
||||
contextReady: true,
|
||||
agentIntegrationReady: true,
|
||||
}).join('\n');
|
||||
|
||||
expect(rendered).toContain('KLO context is ready for agents.');
|
||||
expect(rendered).toContain('klo agent context --json');
|
||||
expect(rendered).toContain('klo serve --mcp stdio --user-id local');
|
||||
expect(rendered).not.toContain('Build KLO context next.');
|
||||
});
|
||||
});
|
||||
104
packages/cli/src/next-steps.ts
Normal file
104
packages/cli/src/next-steps.ts
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
export const KLO_CONTEXT_BUILD_COMMANDS = [
|
||||
{
|
||||
command: 'klo setup context build',
|
||||
description: 'Build agent-ready context from configured primary and context sources',
|
||||
},
|
||||
{
|
||||
command: 'klo status',
|
||||
description: 'Check setup and context readiness',
|
||||
},
|
||||
{
|
||||
command: 'klo setup context status',
|
||||
description: 'Check the setup-managed context build state',
|
||||
},
|
||||
] as const;
|
||||
|
||||
export const KLO_NEXT_STEP_DIRECT_COMMANDS = [
|
||||
{
|
||||
command: 'klo agent context --json',
|
||||
description: 'Verify the project context your agent can read',
|
||||
},
|
||||
{
|
||||
command: 'klo agent tools --json',
|
||||
description: 'List direct CLI tools available to agents',
|
||||
},
|
||||
{
|
||||
command: 'klo sl list',
|
||||
description: 'Inspect generated semantic-layer sources',
|
||||
},
|
||||
{
|
||||
command: 'klo wiki list',
|
||||
description: 'Inspect generated wiki pages',
|
||||
},
|
||||
] as const;
|
||||
|
||||
export const KLO_NEXT_STEP_MCP_COMMANDS = [
|
||||
{
|
||||
command: 'klo serve --mcp stdio --user-id local',
|
||||
description: 'Optional MCP server route for clients that require MCP',
|
||||
},
|
||||
] as const;
|
||||
|
||||
export const KLO_NEXT_STEP_COMMANDS = [...KLO_NEXT_STEP_DIRECT_COMMANDS, ...KLO_NEXT_STEP_MCP_COMMANDS] as const;
|
||||
|
||||
export const KLO_NEXT_STEP_COMMAND_WIDTH = Math.max(
|
||||
...[...KLO_CONTEXT_BUILD_COMMANDS, ...KLO_NEXT_STEP_COMMANDS].map((step) => step.command.length),
|
||||
);
|
||||
|
||||
export interface KloSetupNextStepState {
|
||||
setupReady: boolean;
|
||||
hasContextTargets: boolean;
|
||||
contextReady: boolean;
|
||||
agentIntegrationReady: boolean;
|
||||
}
|
||||
|
||||
function commandLines(commands: ReadonlyArray<{ command: string; description: string }>, indent: string): string[] {
|
||||
return commands.map((step) => `${indent}$ ${step.command.padEnd(KLO_NEXT_STEP_COMMAND_WIDTH)} ${step.description}`);
|
||||
}
|
||||
|
||||
export function formatNextStepLines(indent = ' '): string[] {
|
||||
return [
|
||||
`${indent}KLO context is ready for agents.`,
|
||||
`${indent}Preferred route: CLI + Skills; installed rules call \`klo agent ...\` directly, so no MCP server is required.`,
|
||||
`${indent}Direct CLI checks:`,
|
||||
...commandLines(KLO_NEXT_STEP_DIRECT_COMMANDS, indent),
|
||||
`${indent}Optional MCP:`,
|
||||
...commandLines(KLO_NEXT_STEP_MCP_COMMANDS, indent),
|
||||
];
|
||||
}
|
||||
|
||||
export function formatSetupNextStepLines(state: KloSetupNextStepState, indent = ' '): string[] {
|
||||
if (!state.setupReady) {
|
||||
return [
|
||||
`${indent}Finish setup first.`,
|
||||
`${indent}$ ${'klo setup'.padEnd(KLO_NEXT_STEP_COMMAND_WIDTH)} Resume configuration and validation`,
|
||||
`${indent}$ ${'klo status'.padEnd(KLO_NEXT_STEP_COMMAND_WIDTH)} Check which setup steps still need attention`,
|
||||
];
|
||||
}
|
||||
|
||||
if (!state.hasContextTargets) {
|
||||
return [
|
||||
`${indent}Connect data, then build context.`,
|
||||
`${indent}$ ${'klo setup'.padEnd(KLO_NEXT_STEP_COMMAND_WIDTH)} Add primary or context sources`,
|
||||
`${indent}$ ${'klo status'.padEnd(KLO_NEXT_STEP_COMMAND_WIDTH)} Check setup and context readiness`,
|
||||
];
|
||||
}
|
||||
|
||||
if (!state.contextReady) {
|
||||
return [
|
||||
`${indent}Build KLO context next.`,
|
||||
`${indent}Preferred route: run the CLI build; it covers primary-source scans and context-source ingests.`,
|
||||
...commandLines(KLO_CONTEXT_BUILD_COMMANDS, indent),
|
||||
];
|
||||
}
|
||||
|
||||
if (!state.agentIntegrationReady) {
|
||||
return [
|
||||
`${indent}KLO context is built. Install agent rules when you want your coding agent to use it.`,
|
||||
`${indent}$ ${'klo setup --agents'.padEnd(KLO_NEXT_STEP_COMMAND_WIDTH)} Install CLI-based agent rules`,
|
||||
`${indent}$ ${'klo status'.padEnd(KLO_NEXT_STEP_COMMAND_WIDTH)} Check setup and context readiness`,
|
||||
];
|
||||
}
|
||||
|
||||
return formatNextStepLines(indent);
|
||||
}
|
||||
172
packages/cli/src/project-dir.test.ts
Normal file
172
packages/cli/src/project-dir.test.ts
Normal file
|
|
@ -0,0 +1,172 @@
|
|||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||
import { runKloCli, type KloCliDeps } from './index.js';
|
||||
|
||||
function makeIo() {
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
return {
|
||||
io: {
|
||||
stdout: {
|
||||
write: (chunk: string) => {
|
||||
stdout += chunk;
|
||||
},
|
||||
},
|
||||
stderr: {
|
||||
write: (chunk: string) => {
|
||||
stderr += chunk;
|
||||
},
|
||||
},
|
||||
},
|
||||
stdout: () => stdout,
|
||||
stderr: () => stderr,
|
||||
};
|
||||
}
|
||||
|
||||
describe('project directory defaults', () => {
|
||||
afterEach(() => {
|
||||
delete process.env.KLO_PROJECT_DIR;
|
||||
});
|
||||
|
||||
it('uses KLO_PROJECT_DIR when Commander-dispatched commands omit --project-dir', async () => {
|
||||
process.env.KLO_PROJECT_DIR = '/tmp/klo-env-project';
|
||||
|
||||
const connection = vi.fn(async () => 0);
|
||||
const demo = vi.fn(async () => 0);
|
||||
const doctor = vi.fn(async () => 0);
|
||||
const ingest = vi.fn(async () => 0);
|
||||
const publicIngest = vi.fn(async () => 0);
|
||||
const scan = vi.fn(async () => 0);
|
||||
const serveStdio = vi.fn(async () => 0);
|
||||
const setup = vi.fn(async () => 0);
|
||||
const agent = vi.fn(async () => 0);
|
||||
const deps: KloCliDeps = { agent, connection, demo, doctor, ingest, publicIngest, scan, serveStdio, setup };
|
||||
|
||||
const cases: Array<{
|
||||
argv: string[];
|
||||
spy: ReturnType<typeof vi.fn>;
|
||||
expected: Record<string, unknown>;
|
||||
runnerType: 'cli' | 'serve';
|
||||
}> = [
|
||||
{
|
||||
argv: ['connection', 'list'],
|
||||
spy: connection,
|
||||
expected: { command: 'list', projectDir: '/tmp/klo-env-project' },
|
||||
runnerType: 'cli',
|
||||
},
|
||||
{
|
||||
argv: ['setup', 'demo', 'scan', '--no-input'],
|
||||
spy: demo,
|
||||
expected: { command: 'scan', projectDir: '/tmp/klo-env-project' },
|
||||
runnerType: 'cli',
|
||||
},
|
||||
{
|
||||
argv: ['dev', 'doctor', '--no-input'],
|
||||
spy: doctor,
|
||||
expected: { command: 'project', projectDir: '/tmp/klo-env-project' },
|
||||
runnerType: 'cli',
|
||||
},
|
||||
{
|
||||
argv: ['ingest', 'status', 'run-1'],
|
||||
spy: publicIngest,
|
||||
expected: { command: 'status', projectDir: '/tmp/klo-env-project', runId: 'run-1' },
|
||||
runnerType: 'cli',
|
||||
},
|
||||
{
|
||||
argv: ['setup', 'status'],
|
||||
spy: setup,
|
||||
expected: { command: 'status', projectDir: '/tmp/klo-env-project' },
|
||||
runnerType: 'cli',
|
||||
},
|
||||
{
|
||||
argv: ['dev', 'scan', 'warehouse'],
|
||||
spy: scan,
|
||||
expected: { command: 'run', projectDir: '/tmp/klo-env-project', connectionId: 'warehouse' },
|
||||
runnerType: 'cli',
|
||||
},
|
||||
{
|
||||
argv: ['serve', '--mcp', 'stdio'],
|
||||
spy: serveStdio,
|
||||
expected: { mcp: 'stdio', projectDir: '/tmp/klo-env-project' },
|
||||
runnerType: 'serve',
|
||||
},
|
||||
{
|
||||
argv: ['agent', 'tools', '--json'],
|
||||
spy: agent,
|
||||
expected: { command: 'tools', projectDir: '/tmp/klo-env-project' },
|
||||
runnerType: 'cli',
|
||||
},
|
||||
];
|
||||
|
||||
for (const item of cases) {
|
||||
const testIo = makeIo();
|
||||
await expect(runKloCli(item.argv, testIo.io, deps)).resolves.toBe(0);
|
||||
if (item.runnerType === 'serve') {
|
||||
expect(item.spy).toHaveBeenLastCalledWith(expect.objectContaining(item.expected));
|
||||
} else {
|
||||
expect(item.spy).toHaveBeenLastCalledWith(expect.objectContaining(item.expected), testIo.io);
|
||||
}
|
||||
expect(testIo.stderr()).toBe('');
|
||||
}
|
||||
});
|
||||
|
||||
it('lets explicit global --project-dir override KLO_PROJECT_DIR before and after nested commands', async () => {
|
||||
process.env.KLO_PROJECT_DIR = '/tmp/klo-env-project';
|
||||
|
||||
const scan = vi.fn(async () => 0);
|
||||
const publicIngest = vi.fn(async () => 0);
|
||||
const scanIo = makeIo();
|
||||
const ingestIo = makeIo();
|
||||
|
||||
await expect(
|
||||
runKloCli(['--project-dir', '/tmp/klo-explicit-project', 'dev', 'scan', 'warehouse'], scanIo.io, { scan }),
|
||||
).resolves.toBe(0);
|
||||
await expect(
|
||||
runKloCli(['ingest', 'status', 'run-1', '--project-dir=/tmp/klo-explicit-project'], ingestIo.io, {
|
||||
publicIngest,
|
||||
}),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(scan).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ command: 'run', projectDir: '/tmp/klo-explicit-project' }),
|
||||
scanIo.io,
|
||||
);
|
||||
expect(publicIngest).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ command: 'status', projectDir: '/tmp/klo-explicit-project' }),
|
||||
ingestIo.io,
|
||||
);
|
||||
expect(scanIo.stderr()).toBe('');
|
||||
expect(ingestIo.stderr()).toBe('');
|
||||
});
|
||||
|
||||
it('uses nearest ancestor containing klo.yaml when no explicit or environment project-dir exists', async () => {
|
||||
const { mkdir, realpath, writeFile } = await import('node:fs/promises');
|
||||
const { mkdtemp, rm } = await import('node:fs/promises');
|
||||
const { tmpdir } = await import('node:os');
|
||||
const { join } = await import('node:path');
|
||||
|
||||
const originalCwd = process.cwd();
|
||||
const root = await mkdtemp(join(tmpdir(), 'klo-cli-nearest-project-'));
|
||||
const projectDir = join(root, 'warehouse');
|
||||
const nestedDir = join(projectDir, 'nested', 'deeper');
|
||||
await mkdir(nestedDir, { recursive: true });
|
||||
await writeFile(join(projectDir, 'klo.yaml'), 'project: warehouse\n', 'utf-8');
|
||||
const expectedProjectDir = await realpath(projectDir);
|
||||
|
||||
const scan = vi.fn(async () => 0);
|
||||
const testIo = makeIo();
|
||||
|
||||
try {
|
||||
process.chdir(nestedDir);
|
||||
await expect(runKloCli(['dev', 'scan', 'warehouse'], testIo.io, { scan })).resolves.toBe(0);
|
||||
} finally {
|
||||
process.chdir(originalCwd);
|
||||
await rm(root, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
expect(scan).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ command: 'run', projectDir: expectedProjectDir }),
|
||||
testIo.io,
|
||||
);
|
||||
expect(testIo.stderr()).toBe('');
|
||||
});
|
||||
});
|
||||
5
packages/cli/src/project-dir.ts
Normal file
5
packages/cli/src/project-dir.ts
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
import { resolve } from 'node:path';
|
||||
|
||||
export function resolveProjectDir(projectDir?: string, fallback = '.'): string {
|
||||
return resolve(projectDir ?? fallback);
|
||||
}
|
||||
70
packages/cli/src/project-resolver.test.ts
Normal file
70
packages/cli/src/project-resolver.test.ts
Normal file
|
|
@ -0,0 +1,70 @@
|
|||
import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join, resolve } from 'node:path';
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
||||
|
||||
import { findNearestKloProjectDir, resolveKloProjectDir } from './project-resolver.js';
|
||||
|
||||
describe('resolveKloProjectDir', () => {
|
||||
let tempDir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'klo-project-resolver-'));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(tempDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('prefers an explicit project directory', async () => {
|
||||
const explicit = join(tempDir, 'explicit');
|
||||
const envProject = join(tempDir, 'env');
|
||||
await mkdir(explicit, { recursive: true });
|
||||
await mkdir(envProject, { recursive: true });
|
||||
|
||||
expect(
|
||||
resolveKloProjectDir({
|
||||
explicitProjectDir: explicit,
|
||||
env: { KLO_PROJECT_DIR: envProject },
|
||||
cwd: tempDir,
|
||||
}),
|
||||
).toBe(resolve(explicit));
|
||||
});
|
||||
|
||||
it('uses KLO_PROJECT_DIR when no explicit project directory is set', async () => {
|
||||
const envProject = join(tempDir, 'env-project');
|
||||
await mkdir(envProject, { recursive: true });
|
||||
|
||||
expect(resolveKloProjectDir({ env: { KLO_PROJECT_DIR: envProject }, cwd: tempDir })).toBe(resolve(envProject));
|
||||
});
|
||||
|
||||
it('resolves a relative KLO_PROJECT_DIR value from cwd', () => {
|
||||
expect(resolveKloProjectDir({ env: { KLO_PROJECT_DIR: 'env-project' }, cwd: tempDir })).toBe(
|
||||
resolve(tempDir, 'env-project'),
|
||||
);
|
||||
});
|
||||
|
||||
it('uses the nearest ancestor containing klo.yaml', async () => {
|
||||
const project = join(tempDir, 'warehouse');
|
||||
const nested = join(project, 'nested', 'deeper');
|
||||
await mkdir(nested, { recursive: true });
|
||||
await writeFile(join(project, 'klo.yaml'), 'project: warehouse\n', 'utf-8');
|
||||
|
||||
expect(resolveKloProjectDir({ env: {}, cwd: nested })).toBe(resolve(project));
|
||||
expect(findNearestKloProjectDir(nested)).toBe(resolve(project));
|
||||
});
|
||||
|
||||
it('falls back to the current directory when no project marker exists', () => {
|
||||
expect(resolveKloProjectDir({ env: {}, cwd: tempDir })).toBe(resolve(tempDir));
|
||||
expect(findNearestKloProjectDir(tempDir)).toBeUndefined();
|
||||
});
|
||||
|
||||
it('rejects empty explicit and environment project directory values', () => {
|
||||
expect(() => resolveKloProjectDir({ explicitProjectDir: ' ', cwd: tempDir })).toThrow(
|
||||
'--project-dir requires a value',
|
||||
);
|
||||
expect(() => resolveKloProjectDir({ env: { KLO_PROJECT_DIR: ' ' }, cwd: tempDir })).toThrow(
|
||||
'KLO_PROJECT_DIR must not be empty',
|
||||
);
|
||||
});
|
||||
});
|
||||
56
packages/cli/src/project-resolver.ts
Normal file
56
packages/cli/src/project-resolver.ts
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
import { existsSync } from 'node:fs';
|
||||
import { dirname, join, resolve } from 'node:path';
|
||||
|
||||
export interface KloProjectResolverOptions {
|
||||
explicitProjectDir?: string;
|
||||
env?: Partial<Pick<NodeJS.ProcessEnv, 'KLO_PROJECT_DIR'>>;
|
||||
cwd?: string;
|
||||
}
|
||||
|
||||
function nonEmptyValue(value: string | undefined): string | undefined {
|
||||
if (value === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
const trimmed = value.trim();
|
||||
return trimmed.length > 0 ? value : undefined;
|
||||
}
|
||||
|
||||
export function findNearestKloProjectDir(startDir = process.cwd()): string | undefined {
|
||||
let current = resolve(startDir);
|
||||
|
||||
while (true) {
|
||||
if (existsSync(join(current, 'klo.yaml'))) {
|
||||
return current;
|
||||
}
|
||||
|
||||
const parent = dirname(current);
|
||||
if (parent === current) {
|
||||
return undefined;
|
||||
}
|
||||
current = parent;
|
||||
}
|
||||
}
|
||||
|
||||
export function resolveKloProjectDir(options: KloProjectResolverOptions = {}): string {
|
||||
const cwd = options.cwd ?? process.cwd();
|
||||
|
||||
if (options.explicitProjectDir !== undefined) {
|
||||
const explicit = nonEmptyValue(options.explicitProjectDir);
|
||||
if (!explicit) {
|
||||
throw new Error('--project-dir requires a value');
|
||||
}
|
||||
return resolve(cwd, explicit);
|
||||
}
|
||||
|
||||
const rawEnvProjectDir = options.env ? options.env.KLO_PROJECT_DIR : process.env.KLO_PROJECT_DIR;
|
||||
const envProjectDir = nonEmptyValue(rawEnvProjectDir);
|
||||
if (rawEnvProjectDir !== undefined && envProjectDir === undefined) {
|
||||
throw new Error('KLO_PROJECT_DIR must not be empty');
|
||||
}
|
||||
if (envProjectDir !== undefined) {
|
||||
return resolve(cwd, envProjectDir);
|
||||
}
|
||||
|
||||
const resolvedCwd = resolve(cwd);
|
||||
return findNearestKloProjectDir(resolvedCwd) ?? resolvedCwd;
|
||||
}
|
||||
48
packages/cli/src/prompt-navigation.test.ts
Normal file
48
packages/cli/src/prompt-navigation.test.ts
Normal file
|
|
@ -0,0 +1,48 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import { withMenuOptionSpacing, withMultiselectNavigation, withTextInputNavigation } from './prompt-navigation.js';
|
||||
|
||||
describe('prompt navigation helpers', () => {
|
||||
it('leaves compact single-line menu prompts unchanged', () => {
|
||||
expect(withMenuOptionSpacing('What do you want to do?')).toBe('What do you want to do?');
|
||||
});
|
||||
|
||||
it('adds a blank separator between multiline menu copy and the option list', () => {
|
||||
expect(withMenuOptionSpacing('Which embedding option should KLO use?\n\nKLO uses embeddings for search.')).toBe(
|
||||
'Which embedding option should KLO use?\n\nKLO uses embeddings for search.\n',
|
||||
);
|
||||
});
|
||||
|
||||
it('does not duplicate an existing option-list separator', () => {
|
||||
expect(withMenuOptionSpacing('Question\n\nContext\n')).toBe('Question\n\nContext\n');
|
||||
});
|
||||
|
||||
it('keeps multiselect navigation copy multiline so menu renderers can separate it from options', () => {
|
||||
expect(withMultiselectNavigation('Which sources?')).toBe(
|
||||
'Which sources?\nUse Up/Down to move, Space to select or unselect, Enter to confirm, Escape to go back, or Ctrl+C to exit.',
|
||||
);
|
||||
});
|
||||
|
||||
it('adds a blank separator between text input helper copy and the editable value', () => {
|
||||
expect(
|
||||
withTextInputNavigation(
|
||||
'Name this PostgreSQL connection\nKLO will use this short name in commands and config. You can rename it now.',
|
||||
),
|
||||
).toBe(
|
||||
'Name this PostgreSQL connection\n\nKLO will use this short name in commands and config. You can rename it now.\nPress Escape to go back.\n',
|
||||
);
|
||||
});
|
||||
|
||||
it('adds a blank separator before compact text input values', () => {
|
||||
expect(withTextInputNavigation('Project folder path')).toBe('Project folder path\nPress Escape to go back.\n');
|
||||
});
|
||||
|
||||
it('normalizes already hinted text input prompts without duplicating the hint', () => {
|
||||
expect(
|
||||
withTextInputNavigation(
|
||||
'Name this PostgreSQL connection\nKLO will use this short name in commands and config. You can rename it now.\nPress Escape to go back.',
|
||||
),
|
||||
).toBe(
|
||||
'Name this PostgreSQL connection\n\nKLO will use this short name in commands and config. You can rename it now.\nPress Escape to go back.\n',
|
||||
);
|
||||
});
|
||||
});
|
||||
45
packages/cli/src/prompt-navigation.ts
Normal file
45
packages/cli/src/prompt-navigation.ts
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
const MULTISELECT_MENU_NAVIGATION_HINT =
|
||||
'Use Up/Down to move, Space to select or unselect, Enter to confirm, Escape to go back, or Ctrl+C to exit.';
|
||||
const TEXT_INPUT_NAVIGATION_HINT = 'Press Escape to go back.';
|
||||
|
||||
function removeTrailingBlankLines(message: string): string {
|
||||
return message.replace(/\n+$/, '');
|
||||
}
|
||||
|
||||
function withTextInputBodySpacing(message: string): string {
|
||||
const normalized = removeTrailingBlankLines(message);
|
||||
if (!normalized.includes('\n')) {
|
||||
return normalized;
|
||||
}
|
||||
const [title, ...bodyLines] = normalized.split('\n');
|
||||
if (bodyLines[0] === '') {
|
||||
return normalized;
|
||||
}
|
||||
return `${title}\n\n${bodyLines.join('\n')}`;
|
||||
}
|
||||
|
||||
export function withMenuOptionSpacing(message: string): string {
|
||||
if (!message.includes('\n') || message.endsWith('\n')) {
|
||||
return message;
|
||||
}
|
||||
return `${message}\n`;
|
||||
}
|
||||
|
||||
export function withMenuOptionsSpacing<T extends { message: string }>(options: T): T {
|
||||
return { ...options, message: withMenuOptionSpacing(options.message) };
|
||||
}
|
||||
|
||||
export function withMultiselectNavigation(message: string): string {
|
||||
if (message.includes(MULTISELECT_MENU_NAVIGATION_HINT)) {
|
||||
return message;
|
||||
}
|
||||
return `${message}\n${MULTISELECT_MENU_NAVIGATION_HINT}`;
|
||||
}
|
||||
|
||||
export function withTextInputNavigation(message: string): string {
|
||||
const messageWithoutHint = removeTrailingBlankLines(message)
|
||||
.split('\n')
|
||||
.filter((line) => line !== TEXT_INPUT_NAVIGATION_HINT)
|
||||
.join('\n');
|
||||
return `${withTextInputBodySpacing(messageWithoutHint)}\n${TEXT_INPUT_NAVIGATION_HINT}\n`;
|
||||
}
|
||||
292
packages/cli/src/public-ingest.test.ts
Normal file
292
packages/cli/src/public-ingest.test.ts
Normal file
|
|
@ -0,0 +1,292 @@
|
|||
import { buildDefaultKloProjectConfig, type KloProjectConfig } from '@klo/context/project';
|
||||
import { describe, expect, it, vi } from 'vitest';
|
||||
import { buildPublicIngestPlan, type KloPublicIngestProject, runKloPublicIngest } from './public-ingest.js';
|
||||
|
||||
function makeIo(options: { isTTY?: boolean } = {}) {
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
return {
|
||||
io: {
|
||||
stdout: {
|
||||
isTTY: options.isTTY,
|
||||
write: (chunk: string) => {
|
||||
stdout += chunk;
|
||||
},
|
||||
},
|
||||
stderr: {
|
||||
write: (chunk: string) => {
|
||||
stderr += chunk;
|
||||
},
|
||||
},
|
||||
},
|
||||
stdout: () => stdout,
|
||||
stderr: () => stderr,
|
||||
};
|
||||
}
|
||||
|
||||
function projectWithConnections(connections: KloProjectConfig['connections']): KloPublicIngestProject {
|
||||
return {
|
||||
projectDir: '/tmp/project',
|
||||
config: {
|
||||
...buildDefaultKloProjectConfig('warehouse'),
|
||||
connections,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe('buildPublicIngestPlan', () => {
|
||||
it('plans warehouse connections as scan targets and source connections as source ingest targets', () => {
|
||||
const project = projectWithConnections({
|
||||
warehouse: { driver: 'postgres' },
|
||||
prod_metabase: { driver: 'metabase' },
|
||||
docs: { driver: 'notion' },
|
||||
});
|
||||
|
||||
expect(buildPublicIngestPlan(project, { projectDir: '/tmp/project', all: true })).toEqual({
|
||||
projectDir: '/tmp/project',
|
||||
targets: [
|
||||
{
|
||||
connectionId: 'warehouse',
|
||||
driver: 'postgres',
|
||||
operation: 'scan',
|
||||
debugCommand: 'klo scan warehouse --debug',
|
||||
steps: ['scan'],
|
||||
},
|
||||
{
|
||||
connectionId: 'docs',
|
||||
driver: 'notion',
|
||||
operation: 'source-ingest',
|
||||
adapter: 'notion',
|
||||
debugCommand: 'klo dev ingest run --connection-id docs --adapter notion --debug',
|
||||
steps: ['source-ingest', 'memory-update'],
|
||||
},
|
||||
{
|
||||
connectionId: 'prod_metabase',
|
||||
driver: 'metabase',
|
||||
operation: 'source-ingest',
|
||||
adapter: 'metabase',
|
||||
debugCommand: 'klo dev ingest run --connection-id prod_metabase --adapter metabase --debug',
|
||||
steps: ['source-ingest', 'memory-update'],
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it('rejects bare non-interactive ingest until the interactive confirmation slice exists', () => {
|
||||
const project = projectWithConnections({ warehouse: { driver: 'postgres' } });
|
||||
|
||||
expect(() => buildPublicIngestPlan(project, { projectDir: '/tmp/project', all: false })).toThrow(
|
||||
'klo ingest requires <connectionId> or --all in this release',
|
||||
);
|
||||
});
|
||||
|
||||
it('does not plan PostHog connections as CLI ingest targets', () => {
|
||||
const project = projectWithConnections({ product: { driver: 'posthog' } });
|
||||
|
||||
expect(() =>
|
||||
buildPublicIngestPlan(project, { projectDir: '/tmp/project', targetConnectionId: 'product', all: false }),
|
||||
).toThrow('Connection "product" uses unsupported public ingest driver "posthog"');
|
||||
});
|
||||
});
|
||||
|
||||
describe('runKloPublicIngest', () => {
|
||||
it('runs all independent targets and reports partial failures', async () => {
|
||||
const io = makeIo();
|
||||
const project = projectWithConnections({
|
||||
warehouse: { driver: 'postgres' },
|
||||
prod_metabase: { driver: 'metabase' },
|
||||
});
|
||||
const runScan = vi.fn(async () => 1);
|
||||
const runIngest = vi.fn(async () => 0);
|
||||
|
||||
await expect(
|
||||
runKloPublicIngest(
|
||||
{ command: 'run', projectDir: '/tmp/project', all: true, json: false, inputMode: 'disabled' },
|
||||
io.io,
|
||||
{
|
||||
loadProject: vi.fn(async () => project),
|
||||
runScan,
|
||||
runIngest,
|
||||
},
|
||||
),
|
||||
).resolves.toBe(1);
|
||||
|
||||
expect(runIngest).toHaveBeenCalledWith(
|
||||
{
|
||||
command: 'run',
|
||||
projectDir: '/tmp/project',
|
||||
connectionId: 'prod_metabase',
|
||||
adapter: 'metabase',
|
||||
outputMode: 'plain',
|
||||
inputMode: 'disabled',
|
||||
},
|
||||
expect.anything(),
|
||||
);
|
||||
expect(runScan).toHaveBeenCalledWith(
|
||||
{
|
||||
command: 'run',
|
||||
projectDir: '/tmp/project',
|
||||
connectionId: 'warehouse',
|
||||
mode: 'structural',
|
||||
detectRelationships: false,
|
||||
dryRun: false,
|
||||
},
|
||||
expect.anything(),
|
||||
);
|
||||
expect(io.stdout()).toContain('Ingest finished with partial failures');
|
||||
expect(io.stdout()).toContain('warehouse failed at scan.');
|
||||
expect(io.stdout()).toContain('Debug: klo scan warehouse --debug');
|
||||
});
|
||||
|
||||
it('can request enriched relationship scans for setup-managed context builds', async () => {
|
||||
const io = makeIo();
|
||||
const project = projectWithConnections({ warehouse: { driver: 'postgres' } });
|
||||
const runScan = vi.fn(async () => 0);
|
||||
|
||||
await expect(
|
||||
runKloPublicIngest(
|
||||
{
|
||||
command: 'run',
|
||||
projectDir: '/tmp/project',
|
||||
all: true,
|
||||
json: false,
|
||||
inputMode: 'disabled',
|
||||
scanMode: 'enriched',
|
||||
detectRelationships: true,
|
||||
},
|
||||
io.io,
|
||||
{
|
||||
loadProject: vi.fn(async () => project),
|
||||
runScan,
|
||||
},
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(runScan).toHaveBeenCalledWith(
|
||||
{
|
||||
command: 'run',
|
||||
projectDir: '/tmp/project',
|
||||
connectionId: 'warehouse',
|
||||
mode: 'enriched',
|
||||
detectRelationships: true,
|
||||
dryRun: false,
|
||||
},
|
||||
io.io,
|
||||
);
|
||||
});
|
||||
|
||||
it('prints stable JSON results', async () => {
|
||||
const io = makeIo();
|
||||
const project = projectWithConnections({ warehouse: { driver: 'postgres' } });
|
||||
|
||||
await expect(
|
||||
runKloPublicIngest(
|
||||
{
|
||||
command: 'run',
|
||||
projectDir: '/tmp/project',
|
||||
targetConnectionId: 'warehouse',
|
||||
all: false,
|
||||
json: true,
|
||||
inputMode: 'disabled',
|
||||
},
|
||||
io.io,
|
||||
{
|
||||
loadProject: vi.fn(async () => project),
|
||||
runScan: vi.fn(async () => 0),
|
||||
},
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(JSON.parse(io.stdout())).toMatchObject({
|
||||
plan: { projectDir: '/tmp/project' },
|
||||
results: [{ connectionId: 'warehouse', driver: 'postgres' }],
|
||||
});
|
||||
});
|
||||
|
||||
it('passes dbt source_dir from connection config to runKloIngest', async () => {
|
||||
const runIngest = vi.fn(async () => 0);
|
||||
const io = makeIo();
|
||||
|
||||
await expect(
|
||||
runKloPublicIngest(
|
||||
{
|
||||
command: 'run',
|
||||
projectDir: '/tmp/klo',
|
||||
targetConnectionId: 'analytics_dbt',
|
||||
all: false,
|
||||
json: false,
|
||||
inputMode: 'disabled',
|
||||
},
|
||||
io.io,
|
||||
{
|
||||
loadProject: async () =>
|
||||
({
|
||||
projectDir: '/tmp/klo',
|
||||
config: {
|
||||
connections: {
|
||||
analytics_dbt: {
|
||||
driver: 'dbt',
|
||||
source_dir: '/repo/dbt',
|
||||
},
|
||||
},
|
||||
},
|
||||
}) as never,
|
||||
runIngest,
|
||||
},
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(runIngest).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
command: 'run',
|
||||
connectionId: 'analytics_dbt',
|
||||
adapter: 'dbt',
|
||||
sourceDir: '/repo/dbt',
|
||||
}),
|
||||
io.io,
|
||||
);
|
||||
});
|
||||
|
||||
it('routes public status and watch to the ingest status renderer', async () => {
|
||||
const runIngest = vi.fn(async () => 0);
|
||||
const statusIo = makeIo();
|
||||
const watchIo = makeIo();
|
||||
|
||||
await expect(
|
||||
runKloPublicIngest(
|
||||
{ command: 'status', projectDir: '/tmp/klo', json: false, inputMode: 'disabled' },
|
||||
statusIo.io,
|
||||
{ runIngest },
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
await expect(
|
||||
runKloPublicIngest(
|
||||
{ command: 'watch', projectDir: '/tmp/klo', runId: 'run-1', json: false, inputMode: 'auto' },
|
||||
watchIo.io,
|
||||
{ runIngest },
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(runIngest).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
{
|
||||
command: 'status',
|
||||
projectDir: '/tmp/klo',
|
||||
outputMode: 'plain',
|
||||
inputMode: 'disabled',
|
||||
},
|
||||
statusIo.io,
|
||||
);
|
||||
expect(runIngest).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
{
|
||||
command: 'watch',
|
||||
projectDir: '/tmp/klo',
|
||||
runId: 'run-1',
|
||||
outputMode: 'viz',
|
||||
inputMode: 'auto',
|
||||
},
|
||||
watchIo.io,
|
||||
);
|
||||
});
|
||||
});
|
||||
315
packages/cli/src/public-ingest.ts
Normal file
315
packages/cli/src/public-ingest.ts
Normal file
|
|
@ -0,0 +1,315 @@
|
|||
import { type KloLocalProject, type KloProjectConnectionConfig, loadKloProject } from '@klo/context/project';
|
||||
import type { KloCliIo } from './index.js';
|
||||
import type { KloIngestArgs } from './ingest.js';
|
||||
import type { KloScanArgs } from './scan.js';
|
||||
import { profileMark } from './startup-profile.js';
|
||||
|
||||
profileMark('module:public-ingest');
|
||||
|
||||
export type KloPublicIngestStepName = 'scan' | 'source-ingest' | 'enrich' | 'memory-update';
|
||||
export type KloPublicIngestStepStatus = 'done' | 'skipped' | 'failed' | 'not-run';
|
||||
export type KloPublicIngestInputMode = 'auto' | 'disabled';
|
||||
|
||||
export type KloPublicIngestArgs =
|
||||
| {
|
||||
command: 'run';
|
||||
projectDir: string;
|
||||
targetConnectionId?: string;
|
||||
all: boolean;
|
||||
json: boolean;
|
||||
inputMode: KloPublicIngestInputMode;
|
||||
scanMode?: Extract<KloScanArgs, { command: 'run' }>['mode'];
|
||||
detectRelationships?: boolean;
|
||||
}
|
||||
| {
|
||||
command: 'status' | 'watch';
|
||||
projectDir: string;
|
||||
runId?: string;
|
||||
json: boolean;
|
||||
inputMode: KloPublicIngestInputMode;
|
||||
};
|
||||
|
||||
export interface KloPublicIngestPlanTarget {
|
||||
connectionId: string;
|
||||
driver: string;
|
||||
operation: 'scan' | 'source-ingest';
|
||||
adapter?: string;
|
||||
sourceDir?: string;
|
||||
debugCommand: string;
|
||||
steps: KloPublicIngestStepName[];
|
||||
}
|
||||
|
||||
export interface KloPublicIngestPlan {
|
||||
projectDir: string;
|
||||
targets: KloPublicIngestPlanTarget[];
|
||||
}
|
||||
|
||||
export interface KloPublicIngestTargetResult {
|
||||
connectionId: string;
|
||||
driver: string;
|
||||
steps: Array<{
|
||||
operation: KloPublicIngestStepName;
|
||||
status: KloPublicIngestStepStatus;
|
||||
detail?: string;
|
||||
debugCommand?: string;
|
||||
}>;
|
||||
}
|
||||
|
||||
export type KloPublicIngestProject = Pick<KloLocalProject, 'projectDir' | 'config'>;
|
||||
|
||||
export interface KloPublicIngestDeps {
|
||||
loadProject?: (options: Parameters<typeof loadKloProject>[0]) => Promise<KloPublicIngestProject>;
|
||||
runScan?: (args: KloScanArgs, io: KloCliIo) => Promise<number>;
|
||||
runIngest?: (args: KloIngestArgs, io: KloCliIo) => Promise<number>;
|
||||
}
|
||||
|
||||
const sourceAdapterByDriver = new Map<string, string>([
|
||||
['metabase', 'metabase'],
|
||||
['local_metabase', 'metabase'],
|
||||
['looker', 'looker'],
|
||||
['local_looker', 'looker'],
|
||||
['notion', 'notion'],
|
||||
['metricflow', 'metricflow'],
|
||||
['dbt', 'dbt'],
|
||||
['lookml', 'lookml'],
|
||||
]);
|
||||
|
||||
const warehouseDrivers = new Set([
|
||||
'sqlite',
|
||||
'postgres',
|
||||
'postgresql',
|
||||
'mysql',
|
||||
'clickhouse',
|
||||
'sqlserver',
|
||||
'bigquery',
|
||||
'snowflake',
|
||||
]);
|
||||
|
||||
function normalizedDriver(connection: KloProjectConnectionConfig): string {
|
||||
return String(connection.driver ?? '')
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
}
|
||||
|
||||
function sourceDirForConnection(connection: KloProjectConnectionConfig): string | undefined {
|
||||
const value = connection.source_dir ?? connection.sourceDir;
|
||||
return typeof value === 'string' && value.trim().length > 0 ? value.trim() : undefined;
|
||||
}
|
||||
|
||||
function targetForConnection(connectionId: string, connection: KloProjectConnectionConfig): KloPublicIngestPlanTarget {
|
||||
const driver = normalizedDriver(connection);
|
||||
const adapter = sourceAdapterByDriver.get(driver);
|
||||
const sourceDir = sourceDirForConnection(connection);
|
||||
if (adapter) {
|
||||
return {
|
||||
connectionId,
|
||||
driver,
|
||||
operation: 'source-ingest',
|
||||
adapter,
|
||||
...(sourceDir ? { sourceDir } : {}),
|
||||
debugCommand: `klo dev ingest run --connection-id ${connectionId} --adapter ${adapter} --debug`,
|
||||
steps: ['source-ingest', 'memory-update'],
|
||||
};
|
||||
}
|
||||
|
||||
if (warehouseDrivers.has(driver)) {
|
||||
return {
|
||||
connectionId,
|
||||
driver,
|
||||
operation: 'scan',
|
||||
debugCommand: `klo scan ${connectionId} --debug`,
|
||||
steps: ['scan'],
|
||||
};
|
||||
}
|
||||
|
||||
throw new Error(`Connection "${connectionId}" uses unsupported public ingest driver "${driver || 'unknown'}"`);
|
||||
}
|
||||
|
||||
export function buildPublicIngestPlan(
|
||||
project: KloPublicIngestProject,
|
||||
args: { projectDir: string; targetConnectionId?: string; all: boolean },
|
||||
): KloPublicIngestPlan {
|
||||
if (!args.all && !args.targetConnectionId) {
|
||||
throw new Error('klo ingest requires <connectionId> or --all in this release');
|
||||
}
|
||||
|
||||
const entries = Object.entries(project.config.connections).sort(([a], [b]) => a.localeCompare(b));
|
||||
const selected = args.all ? entries : entries.filter(([connectionId]) => connectionId === args.targetConnectionId);
|
||||
|
||||
if (!args.all && selected.length === 0) {
|
||||
throw new Error(`Connection "${args.targetConnectionId}" is not configured in klo.yaml`);
|
||||
}
|
||||
if (selected.length === 0) {
|
||||
throw new Error('No configured connections are eligible for ingest');
|
||||
}
|
||||
|
||||
const targets = selected.map(([connectionId, connection]) => targetForConnection(connectionId, connection));
|
||||
return {
|
||||
projectDir: args.projectDir,
|
||||
targets: [...targets.filter((t) => t.operation === 'scan'), ...targets.filter((t) => t.operation === 'source-ingest')],
|
||||
};
|
||||
}
|
||||
|
||||
function defaultSteps(target: KloPublicIngestPlanTarget): KloPublicIngestTargetResult['steps'] {
|
||||
return [
|
||||
{
|
||||
operation: 'scan',
|
||||
status: target.steps.includes('scan') ? 'not-run' : 'skipped',
|
||||
...(target.operation === 'scan' ? { debugCommand: target.debugCommand } : {}),
|
||||
},
|
||||
{
|
||||
operation: 'source-ingest',
|
||||
status: target.steps.includes('source-ingest') ? 'not-run' : 'skipped',
|
||||
...(target.operation === 'source-ingest' ? { debugCommand: target.debugCommand } : {}),
|
||||
},
|
||||
{ operation: 'enrich', status: 'skipped' },
|
||||
{
|
||||
operation: 'memory-update',
|
||||
status: target.steps.includes('memory-update') ? 'not-run' : 'skipped',
|
||||
...(target.operation === 'source-ingest' ? { debugCommand: target.debugCommand } : {}),
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
function markTargetResult(target: KloPublicIngestPlanTarget, status: 'done' | 'failed'): KloPublicIngestTargetResult {
|
||||
const failedOperation = target.operation === 'scan' ? 'scan' : 'source-ingest';
|
||||
return {
|
||||
connectionId: target.connectionId,
|
||||
driver: target.driver,
|
||||
steps: defaultSteps(target).map((step) => {
|
||||
if (!target.steps.includes(step.operation)) {
|
||||
return step;
|
||||
}
|
||||
if (status === 'done') {
|
||||
return { ...step, status: 'done' };
|
||||
}
|
||||
if (step.operation === failedOperation) {
|
||||
return { ...step, status: 'failed', detail: `${target.connectionId} failed at ${failedOperation}.` };
|
||||
}
|
||||
return { ...step, status: 'not-run' };
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
function resultFailed(result: KloPublicIngestTargetResult): boolean {
|
||||
return result.steps.some((step) => step.status === 'failed');
|
||||
}
|
||||
|
||||
function stepStatus(result: KloPublicIngestTargetResult, operation: KloPublicIngestStepName): string {
|
||||
return result.steps.find((step) => step.operation === operation)?.status ?? 'not-run';
|
||||
}
|
||||
|
||||
function renderPlainResults(results: KloPublicIngestTargetResult[], io: KloCliIo): void {
|
||||
const failures = results.filter(resultFailed);
|
||||
io.stdout.write(failures.length > 0 ? 'Ingest finished with partial failures\n' : 'Ingest finished\n');
|
||||
io.stdout.write('\n');
|
||||
io.stdout.write('Source Scan Source ingest Enrich Memory update\n');
|
||||
for (const result of results) {
|
||||
io.stdout.write(
|
||||
`${result.connectionId.padEnd(14)} ${stepStatus(result, 'scan').padEnd(9)} ${stepStatus(
|
||||
result,
|
||||
'source-ingest',
|
||||
).padEnd(14)} ${stepStatus(result, 'enrich').padEnd(8)} ${stepStatus(result, 'memory-update')}\n`,
|
||||
);
|
||||
}
|
||||
|
||||
if (failures.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
io.stdout.write('\nFailed sources:\n');
|
||||
for (const result of failures) {
|
||||
const failedStep = result.steps.find((step) => step.status === 'failed');
|
||||
if (!failedStep) {
|
||||
continue;
|
||||
}
|
||||
io.stdout.write(` ${failedStep.detail ?? `${result.connectionId} failed.`}\n`);
|
||||
if (failedStep.debugCommand) {
|
||||
io.stdout.write(` Debug: ${failedStep.debugCommand}\n`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function hasInteractiveInput(io: KloCliIo): boolean {
|
||||
const stdin = (io as { stdin?: { isTTY?: boolean; setRawMode?: (value: boolean) => void } }).stdin;
|
||||
return stdin?.isTTY === true && typeof stdin.setRawMode === 'function';
|
||||
}
|
||||
|
||||
function sourceIngestOutputMode(args: Extract<KloPublicIngestArgs, { command: 'run' }>, io: KloCliIo): 'plain' | 'viz' {
|
||||
return args.inputMode === 'auto' && io.stdout.isTTY === true && hasInteractiveInput(io) ? 'viz' : 'plain';
|
||||
}
|
||||
|
||||
export async function executePublicIngestTarget(
|
||||
target: KloPublicIngestPlanTarget,
|
||||
args: Extract<KloPublicIngestArgs, { command: 'run' }>,
|
||||
io: KloCliIo,
|
||||
deps: KloPublicIngestDeps,
|
||||
): Promise<KloPublicIngestTargetResult> {
|
||||
if (target.operation === 'scan') {
|
||||
const { runKloScan } = await import('./scan.js');
|
||||
const exitCode = await (deps.runScan ?? runKloScan)(
|
||||
{
|
||||
command: 'run',
|
||||
projectDir: args.projectDir,
|
||||
connectionId: target.connectionId,
|
||||
mode: args.scanMode ?? 'structural',
|
||||
detectRelationships: args.detectRelationships ?? false,
|
||||
dryRun: false,
|
||||
},
|
||||
io,
|
||||
);
|
||||
return markTargetResult(target, exitCode === 0 ? 'done' : 'failed');
|
||||
}
|
||||
|
||||
const { runKloIngest } = await import('./ingest.js');
|
||||
const exitCode = await (deps.runIngest ?? runKloIngest)(
|
||||
{
|
||||
command: 'run',
|
||||
projectDir: args.projectDir,
|
||||
connectionId: target.connectionId,
|
||||
adapter: target.adapter ?? target.driver,
|
||||
...(target.sourceDir ? { sourceDir: target.sourceDir } : {}),
|
||||
outputMode: sourceIngestOutputMode(args, io),
|
||||
inputMode: args.inputMode,
|
||||
},
|
||||
io,
|
||||
);
|
||||
return markTargetResult(target, exitCode === 0 ? 'done' : 'failed');
|
||||
}
|
||||
|
||||
export async function runKloPublicIngest(
|
||||
args: KloPublicIngestArgs,
|
||||
io: KloCliIo,
|
||||
deps: KloPublicIngestDeps = {},
|
||||
): Promise<number> {
|
||||
if (args.command !== 'run') {
|
||||
const { runKloIngest } = await import('./ingest.js');
|
||||
return await (deps.runIngest ?? runKloIngest)(
|
||||
{
|
||||
command: args.command,
|
||||
projectDir: args.projectDir,
|
||||
...(args.runId ? { runId: args.runId } : {}),
|
||||
outputMode: args.json ? 'json' : args.command === 'watch' ? 'viz' : 'plain',
|
||||
inputMode: args.inputMode,
|
||||
},
|
||||
io,
|
||||
);
|
||||
}
|
||||
|
||||
const loadProject = deps.loadProject ?? loadKloProject;
|
||||
const project = await loadProject({ projectDir: args.projectDir });
|
||||
const plan = buildPublicIngestPlan(project, args);
|
||||
const results: KloPublicIngestTargetResult[] = [];
|
||||
|
||||
for (const target of plan.targets) {
|
||||
results.push(await executePublicIngestTarget(target, args, io, deps));
|
||||
}
|
||||
|
||||
if (args.json) {
|
||||
io.stdout.write(`${JSON.stringify({ plan, results }, null, 2)}\n`);
|
||||
} else {
|
||||
renderPlainResults(results, io);
|
||||
}
|
||||
|
||||
return results.some(resultFailed) ? 1 : 0;
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Add a link
Reference in a new issue