Initial open-source release

This commit is contained in:
Andrey Avtomonov 2026-05-10 23:12:26 +02:00
commit 1a42152e6f
1199 changed files with 257054 additions and 0 deletions

View file

@ -0,0 +1,108 @@
import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import {
KLO_AGENT_MAX_ROWS_CAP,
createKloAgentRuntime,
parseAgentMaxRows,
readAgentJsonFile,
writeAgentJson,
writeAgentJsonError,
} from './agent-runtime.js';
function makeIo() {
let stdout = '';
let stderr = '';
return {
io: {
stdout: { write: (chunk: string) => (stdout += chunk) },
stderr: { write: (chunk: string) => (stderr += chunk) },
},
stdout: () => stdout,
stderr: () => stderr,
};
}
describe('agent runtime helpers', () => {
let tempDir: string;
beforeEach(async () => {
tempDir = await mkdtemp(join(tmpdir(), 'klo-agent-runtime-'));
});
afterEach(async () => {
await rm(tempDir, { recursive: true, force: true });
});
it('writes JSON success and error envelopes without color or spinners', () => {
const successIo = makeIo();
const errorIo = makeIo();
writeAgentJson(successIo.io, { ok: true });
writeAgentJsonError(errorIo.io, 'missing source', { code: 'NOT_FOUND' });
expect(JSON.parse(successIo.stdout())).toEqual({ ok: true });
expect(successIo.stderr()).toBe('');
expect(JSON.parse(errorIo.stderr())).toEqual({
ok: false,
error: { message: 'missing source', code: 'NOT_FOUND' },
});
expect(errorIo.stdout()).toBe('');
});
it('reads JSON query files as objects', async () => {
const path = join(tempDir, 'query.json');
await writeFile(path, '{"measures":["revenue"],"limit":50}', 'utf-8');
await expect(readAgentJsonFile(path)).resolves.toEqual({ measures: ['revenue'], limit: 50 });
});
it('rejects non-object JSON query files', async () => {
const path = join(tempDir, 'query.json');
await writeFile(path, '["revenue"]', 'utf-8');
await expect(readAgentJsonFile(path)).rejects.toThrow('must contain a JSON object');
});
it('requires positive row limits and enforces the agent cap', () => {
expect(parseAgentMaxRows(100)).toBe(100);
expect(() => parseAgentMaxRows(undefined)).toThrow('maxRows is required');
expect(() => parseAgentMaxRows(0)).toThrow('positive integer');
expect(() => parseAgentMaxRows(KLO_AGENT_MAX_ROWS_CAP + 1)).toThrow(String(KLO_AGENT_MAX_ROWS_CAP));
});
it('constructs local context ports with semantic compute and query executor', async () => {
const project = {
projectDir: tempDir,
configPath: join(tempDir, 'klo.yaml'),
config: { project: 'revenue', connections: {} },
coreConfig: {},
git: {},
fileStore: {},
} as never;
const ports = { knowledge: {}, semanticLayer: {} } as never;
const semanticLayerCompute = { query: vi.fn(), validateSources: vi.fn(), generateSources: vi.fn() };
const queryExecutor = { execute: vi.fn() };
const loadProject = vi.fn(async () => project);
const createContextTools = vi.fn(() => ports);
await expect(
createKloAgentRuntime(
{ projectDir: tempDir, enableSemanticCompute: true, enableQueryExecution: true },
{
loadProject,
createContextTools,
createSemanticLayerCompute: () => semanticLayerCompute,
createQueryExecutor: () => queryExecutor,
},
),
).resolves.toMatchObject({ project, ports, queryExecutor });
expect(loadProject).toHaveBeenCalledWith({ projectDir: tempDir });
expect(createContextTools).toHaveBeenCalledWith(project, {
semanticLayerCompute,
queryExecutor,
});
});
});

View file

@ -0,0 +1,81 @@
import { readFile } from 'node:fs/promises';
import { createDefaultLocalQueryExecutor, type KloSqlQueryExecutorPort } from '@klo/context/connections';
import { createPythonSemanticLayerComputePort, type KloSemanticLayerComputePort } from '@klo/context/daemon';
import { createLocalProjectMcpContextPorts, type KloMcpContextPorts } from '@klo/context/mcp';
import { type KloLocalProject, loadKloProject } from '@klo/context/project';
import type { KloCliIo } from './cli-runtime.js';
export const KLO_AGENT_MAX_ROWS_CAP = 1000;
export interface KloAgentRuntimeOptions {
projectDir: string;
enableSemanticCompute: boolean;
enableQueryExecution: boolean;
}
export interface KloAgentRuntime {
project: KloLocalProject;
ports: KloMcpContextPorts;
semanticLayerCompute?: KloSemanticLayerComputePort;
queryExecutor?: KloSqlQueryExecutorPort;
}
export interface KloAgentRuntimeDeps {
loadProject?: typeof loadKloProject;
createContextTools?: typeof createLocalProjectMcpContextPorts;
createSemanticLayerCompute?: () => KloSemanticLayerComputePort;
createQueryExecutor?: () => KloSqlQueryExecutorPort;
}
export function writeAgentJson(io: KloCliIo, value: unknown): void {
io.stdout.write(`${JSON.stringify(value, null, 2)}\n`);
}
export function writeAgentJsonError(
io: KloCliIo,
message: string,
detail: Record<string, unknown> = {},
): void {
io.stderr.write(`${JSON.stringify({ ok: false, error: { message, ...detail } }, null, 2)}\n`);
}
export async function readAgentJsonFile(path: string): Promise<Record<string, unknown>> {
const parsed = JSON.parse(await readFile(path, 'utf-8')) as unknown;
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
throw new Error(`${path} must contain a JSON object.`);
}
return parsed as Record<string, unknown>;
}
export function parseAgentMaxRows(value: number | undefined): number {
if (!Number.isInteger(value) || value === undefined || value <= 0) {
throw new Error('maxRows is required and must be a positive integer.');
}
if (value > KLO_AGENT_MAX_ROWS_CAP) {
throw new Error(`maxRows must be less than or equal to ${KLO_AGENT_MAX_ROWS_CAP}.`);
}
return value;
}
export async function createKloAgentRuntime(
options: KloAgentRuntimeOptions,
deps: KloAgentRuntimeDeps = {},
): Promise<KloAgentRuntime> {
const project = await (deps.loadProject ?? loadKloProject)({ projectDir: options.projectDir });
const semanticLayerCompute = options.enableSemanticCompute
? (deps.createSemanticLayerCompute ?? createPythonSemanticLayerComputePort)()
: undefined;
const queryExecutor = options.enableQueryExecution
? (deps.createQueryExecutor ?? createDefaultLocalQueryExecutor)()
: undefined;
const ports = (deps.createContextTools ?? createLocalProjectMcpContextPorts)(project, {
...(semanticLayerCompute ? { semanticLayerCompute } : {}),
...(queryExecutor ? { queryExecutor } : {}),
});
return {
project,
ports,
...(semanticLayerCompute ? { semanticLayerCompute } : {}),
...(queryExecutor ? { queryExecutor } : {}),
};
}

View file

@ -0,0 +1,51 @@
import { describe, expect, it } from 'vitest';
import {
isMissingProjectConfigError,
missingConnectionSlSearchReadiness,
missingProjectSlSearchReadiness,
noConnectionsSlSearchReadiness,
noIndexedSourcesSlSearchReadiness,
} from './agent-search-readiness.js';
describe('agent semantic-layer search readiness guidance', () => {
it('formats missing project guidance with exact recovery commands', () => {
expect(missingProjectSlSearchReadiness('/tmp/klo-search', 'gross revenue')).toEqual({
code: 'agent_sl_search_missing_project',
message: 'Semantic-layer search needs an initialized KLO project at /tmp/klo-search.',
nextSteps: [
'klo demo',
'klo setup --project-dir /tmp/klo-search',
'klo ingest <connection>',
'klo agent sl list --json --query "gross revenue" --project-dir /tmp/klo-search',
],
});
});
it('formats no-connection and no-index guidance without hiding the project path', () => {
expect(noConnectionsSlSearchReadiness('/tmp/klo-search', 'revenue')).toMatchObject({
code: 'agent_sl_search_no_connections',
message: 'Semantic-layer search found no configured connections in /tmp/klo-search.',
});
expect(noIndexedSourcesSlSearchReadiness('/tmp/klo-search', 'orders')).toMatchObject({
code: 'agent_sl_search_no_indexed_sources',
message: 'Semantic-layer search found no indexed semantic-layer sources in /tmp/klo-search.',
});
});
it('formats unknown connection guidance', () => {
expect(missingConnectionSlSearchReadiness('/tmp/klo-search', 'warehouse', 'revenue')).toMatchObject({
code: 'agent_sl_search_unknown_connection',
message: 'Semantic-layer search connection "warehouse" is not configured in /tmp/klo-search.',
});
});
it('detects missing klo.yaml read errors', () => {
const error = Object.assign(new Error('ENOENT: no such file or directory'), {
code: 'ENOENT',
path: '/tmp/klo-search/klo.yaml',
});
expect(isMissingProjectConfigError(error)).toBe(true);
expect(isMissingProjectConfigError(new Error('other'))).toBe(false);
});
});

View file

@ -0,0 +1,94 @@
export type KloAgentSlSearchReadinessCode =
| 'agent_sl_search_missing_project'
| 'agent_sl_search_no_connections'
| 'agent_sl_search_unknown_connection'
| 'agent_sl_search_no_indexed_sources';
export interface KloAgentSlSearchReadinessDetail {
code: KloAgentSlSearchReadinessCode;
message: string;
nextSteps: string[];
}
function queryForCommand(query: string | undefined): string {
const trimmed = query?.trim();
return trimmed && trimmed.length > 0 ? trimmed : 'revenue';
}
function projectSearchCommand(projectDir: string, query: string | undefined): string {
return `klo agent sl list --json --query ${JSON.stringify(queryForCommand(query))} --project-dir ${projectDir}`;
}
function baseNextSteps(projectDir: string, query: string | undefined): string[] {
return [
'klo demo',
`klo setup --project-dir ${projectDir}`,
'klo ingest <connection>',
projectSearchCommand(projectDir, query),
];
}
export function missingProjectSlSearchReadiness(
projectDir: string,
query: string | undefined,
): KloAgentSlSearchReadinessDetail {
return {
code: 'agent_sl_search_missing_project',
message: `Semantic-layer search needs an initialized KLO project at ${projectDir}.`,
nextSteps: baseNextSteps(projectDir, query),
};
}
export function noConnectionsSlSearchReadiness(
projectDir: string,
query: string | undefined,
): KloAgentSlSearchReadinessDetail {
return {
code: 'agent_sl_search_no_connections',
message: `Semantic-layer search found no configured connections in ${projectDir}.`,
nextSteps: baseNextSteps(projectDir, query),
};
}
export function missingConnectionSlSearchReadiness(
projectDir: string,
connectionId: string,
query: string | undefined,
): KloAgentSlSearchReadinessDetail {
return {
code: 'agent_sl_search_unknown_connection',
message: `Semantic-layer search connection "${connectionId}" is not configured in ${projectDir}.`,
nextSteps: baseNextSteps(projectDir, query),
};
}
export function noIndexedSourcesSlSearchReadiness(
projectDir: string,
query: string | undefined,
): KloAgentSlSearchReadinessDetail {
return {
code: 'agent_sl_search_no_indexed_sources',
message: `Semantic-layer search found no indexed semantic-layer sources in ${projectDir}.`,
nextSteps: baseNextSteps(projectDir, query),
};
}
function errorCode(error: unknown): string | undefined {
if (typeof error !== 'object' || error === null || !('code' in error)) {
return undefined;
}
const code = (error as { code?: unknown }).code;
return typeof code === 'string' ? code : undefined;
}
function errorPath(error: unknown): string | undefined {
if (typeof error !== 'object' || error === null || !('path' in error)) {
return undefined;
}
const path = (error as { path?: unknown }).path;
return typeof path === 'string' ? path : undefined;
}
export function isMissingProjectConfigError(error: unknown): boolean {
return errorCode(error) === 'ENOENT' && (errorPath(error)?.endsWith('klo.yaml') ?? false);
}

View file

@ -0,0 +1,393 @@
import { mkdtemp, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { buildDefaultKloProjectConfig } from '@klo/context/project';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { runKloAgent } from './agent.js';
import type { KloAgentRuntime } from './agent-runtime.js';
function makeIo() {
let stdout = '';
let stderr = '';
return {
io: {
stdout: { write: (chunk: string) => (stdout += chunk) },
stderr: { write: (chunk: string) => (stderr += chunk) },
},
stdout: () => stdout,
stderr: () => stderr,
};
}
function runtime(overrides: Record<string, unknown> = {}): KloAgentRuntime {
const config = buildDefaultKloProjectConfig('revenue');
return {
project: {
projectDir: '/tmp/revenue',
configPath: '/tmp/revenue/klo.yaml',
config: {
...config,
connections: {
warehouse: { driver: 'sqlite', path: 'warehouse.sqlite', readonly: true as const },
},
},
coreConfig: {} as KloAgentRuntime['project']['coreConfig'],
git: {} as KloAgentRuntime['project']['git'],
fileStore: {} as KloAgentRuntime['project']['fileStore'],
},
ports: {
connections: { list: vi.fn(async () => [{ id: 'warehouse', name: 'warehouse', connectionType: 'sqlite' }]) },
semanticLayer: {
listSources: vi.fn(async () => ({
sources: [
{
connectionId: 'warehouse',
connectionName: 'warehouse',
name: 'orders',
columnCount: 2,
measureCount: 1,
joinCount: 0,
},
],
totalSources: 1,
})),
readSource: vi.fn(async () => ({ sourceName: 'orders', yaml: 'name: orders\n' })),
writeSource: vi.fn(async () => ({ success: true, sourceName: 'orders' })),
validate: vi.fn(async () => ({ success: true, errors: [], warnings: [] })),
query: vi.fn(async () => ({ sql: 'select 1', headers: ['x'], rows: [[1]], totalRows: 1, plan: {} })),
},
knowledge: {
search: vi.fn(async () => ({
results: [
{
key: 'page-1',
path: 'knowledge/global/page-1.md',
scope: 'GLOBAL' as const,
summary: 'Revenue logic',
score: 0.9,
matchReasons: ['lexical' as const],
},
],
totalFound: 1,
})),
read: vi.fn(async () => ({
key: 'page-1',
scope: 'GLOBAL' as const,
summary: 'Revenue logic',
content: 'Use net revenue.',
})),
write: vi.fn(async () => ({ success: true, key: 'page-1', action: 'created' as const })),
},
},
queryExecutor: {
execute: vi.fn(async () => ({ headers: ['x'], rows: [[1]], totalRows: 1, command: 'SELECT', rowCount: 1 })),
},
...overrides,
};
}
function runtimeWithoutConnections(): KloAgentRuntime {
const base = runtime();
return {
...base,
project: {
...base.project,
config: {
...base.project.config,
connections: {},
},
},
ports: {
...base.ports,
semanticLayer: {
...base.ports.semanticLayer!,
listSources: vi.fn(async () => ({ sources: [], totalSources: 0 })),
},
},
};
}
describe('runKloAgent', () => {
let tempDir: string;
beforeEach(async () => {
tempDir = await mkdtemp(join(tmpdir(), 'klo-agent-'));
});
afterEach(async () => {
await rm(tempDir, { recursive: true, force: true });
});
it('prints tool discovery with every stable command', async () => {
const io = makeIo();
await expect(runKloAgent({ command: 'tools', projectDir: tempDir, json: true }, io.io)).resolves.toBe(0);
const body = JSON.parse(io.stdout());
expect(body.projectDir).toBe(tempDir);
expect(body.tools.map((tool: { name: string }) => tool.name)).toEqual([
'context',
'sl.list',
'sl.read',
'sl.query',
'wiki.search',
'wiki.read',
'sql.execute',
]);
expect(io.stderr()).toBe('');
});
it('prints project context from setup status, connections, and SL summaries', async () => {
const io = makeIo();
const createRuntime = vi.fn(async () => runtime());
const readSetupStatus = vi.fn(async () => ({ project: { path: tempDir, ready: true }, agents: [] }));
await expect(
runKloAgent({ command: 'context', projectDir: tempDir, json: true }, io.io, { createRuntime, readSetupStatus }),
).resolves.toBe(0);
expect(JSON.parse(io.stdout())).toMatchObject({
projectDir: tempDir,
status: { project: { ready: true } },
connections: [{ id: 'warehouse' }],
semanticLayer: { totalSources: 1 },
});
});
it('dispatches SL list, SL read, wiki search, and wiki read through local ports', async () => {
for (const args of [
{ command: 'sl-list' as const, projectDir: tempDir, json: true as const, connectionId: 'warehouse' },
{
command: 'sl-read' as const,
projectDir: tempDir,
json: true as const,
connectionId: 'warehouse',
sourceName: 'orders',
},
{ command: 'wiki-search' as const, projectDir: tempDir, json: true as const, query: 'revenue', limit: 10 },
{ command: 'wiki-read' as const, projectDir: tempDir, json: true as const, pageId: 'page-1' },
]) {
const io = makeIo();
await expect(runKloAgent(args, io.io, { createRuntime: async () => runtime() })).resolves.toBe(0);
expect(JSON.parse(io.stdout())).toBeTruthy();
expect(io.stderr()).toBe('');
}
});
it('prints wiki hybrid search metadata from the hidden agent wiki search command', async () => {
const fakeRuntime = runtime();
const knowledge = fakeRuntime.ports.knowledge;
if (!knowledge) {
throw new Error('Expected runtime knowledge port');
}
fakeRuntime.ports.knowledge = {
...knowledge,
search: vi.fn(async () => ({
results: [
{
key: 'metrics/revenue',
path: 'knowledge/global/metrics/revenue.md',
scope: 'GLOBAL' as const,
summary: 'Revenue metric definition',
score: 0.02459016393442623,
matchReasons: ['lexical' as const, 'token' as const],
},
],
totalFound: 1,
})),
};
const io = makeIo();
await expect(
runKloAgent({ command: 'wiki-search', projectDir: tempDir, json: true, query: 'paid order', limit: 5 }, io.io, {
createRuntime: async () => fakeRuntime,
}),
).resolves.toBe(0);
expect(JSON.parse(io.stdout())).toEqual({
results: [
expect.objectContaining({
key: 'metrics/revenue',
path: 'knowledge/global/metrics/revenue.md',
matchReasons: ['lexical', 'token'],
}),
],
totalFound: 1,
});
});
it('executes SL queries from a JSON query file', async () => {
const queryFile = join(tempDir, 'sl-query.json');
const io = makeIo();
await writeFile(queryFile, '{"measures":["total_revenue"],"dimensions":[]}', 'utf-8');
await expect(
runKloAgent(
{
command: 'sl-query',
projectDir: tempDir,
json: true,
connectionId: 'warehouse',
queryFile,
execute: true,
maxRows: 100,
},
io.io,
{ createRuntime: async () => runtime() },
),
).resolves.toBe(0);
expect(JSON.parse(io.stdout())).toMatchObject({ sql: 'select 1', rows: [[1]] });
});
it('executes read-only SQL from a SQL file with an explicit row limit', async () => {
const sqlFile = join(tempDir, 'query.sql');
const fakeRuntime = runtime();
const io = makeIo();
await writeFile(sqlFile, 'select 1', 'utf-8');
await expect(
runKloAgent(
{
command: 'sql-execute',
projectDir: tempDir,
json: true,
connectionId: 'warehouse',
sqlFile,
maxRows: 100,
},
io.io,
{ createRuntime: async () => fakeRuntime as never },
),
).resolves.toBe(0);
expect(fakeRuntime.queryExecutor?.execute).toHaveBeenCalledWith({
connectionId: 'warehouse',
projectDir: '/tmp/revenue',
connection: { driver: 'sqlite', path: 'warehouse.sqlite', readonly: true },
sql: 'select 1',
maxRows: 100,
});
});
it('prints guided JSON when semantic-layer search runs outside a project', async () => {
const io = makeIo();
const missingProjectError = Object.assign(new Error('ENOENT: no such file or directory'), {
code: 'ENOENT',
path: join(tempDir, 'klo.yaml'),
});
await expect(
runKloAgent(
{ command: 'sl-list', projectDir: tempDir, json: true, query: 'gross revenue' },
io.io,
{ createRuntime: vi.fn(async () => Promise.reject(missingProjectError)) },
),
).resolves.toBe(1);
expect(JSON.parse(io.stderr())).toEqual({
ok: false,
error: {
code: 'agent_sl_search_missing_project',
message: `Semantic-layer search needs an initialized KLO project at ${tempDir}.`,
nextSteps: [
'klo demo',
`klo setup --project-dir ${tempDir}`,
'klo ingest <connection>',
`klo agent sl list --json --query "gross revenue" --project-dir ${tempDir}`,
],
},
});
expect(io.stdout()).toBe('');
});
it('prints guided JSON when semantic-layer search has no configured connections', async () => {
const io = makeIo();
await expect(
runKloAgent(
{ command: 'sl-list', projectDir: tempDir, json: true, query: 'revenue' },
io.io,
{ createRuntime: async () => runtimeWithoutConnections() },
),
).resolves.toBe(1);
expect(JSON.parse(io.stderr())).toMatchObject({
ok: false,
error: {
code: 'agent_sl_search_no_connections',
message: `Semantic-layer search found no configured connections in ${tempDir}.`,
nextSteps: [
'klo demo',
`klo setup --project-dir ${tempDir}`,
'klo ingest <connection>',
`klo agent sl list --json --query "revenue" --project-dir ${tempDir}`,
],
},
});
});
it('prints guided JSON when semantic-layer search asks for an unknown connection', async () => {
const io = makeIo();
await expect(
runKloAgent(
{ command: 'sl-list', projectDir: tempDir, json: true, connectionId: 'missing', query: 'revenue' },
io.io,
{ createRuntime: async () => runtime() },
),
).resolves.toBe(1);
expect(JSON.parse(io.stderr())).toMatchObject({
ok: false,
error: {
code: 'agent_sl_search_unknown_connection',
message: `Semantic-layer search connection "missing" is not configured in ${tempDir}.`,
},
});
});
it('prints guided JSON when semantic-layer search has no indexed sources', async () => {
const fakeRuntime = runtime();
const semanticLayer = fakeRuntime.ports.semanticLayer!;
fakeRuntime.ports.semanticLayer = {
...semanticLayer,
listSources: vi.fn(async () => ({ sources: [], totalSources: 0 })),
};
const io = makeIo();
await expect(
runKloAgent(
{ command: 'sl-list', projectDir: tempDir, json: true, connectionId: 'warehouse', query: 'revenue' },
io.io,
{ createRuntime: async () => fakeRuntime },
),
).resolves.toBe(1);
expect(JSON.parse(io.stderr())).toMatchObject({
ok: false,
error: {
code: 'agent_sl_search_no_indexed_sources',
message: `Semantic-layer search found no indexed semantic-layer sources in ${tempDir}.`,
},
});
});
it('returns JSON errors when required ports or records are missing', async () => {
const io = makeIo();
await expect(
runKloAgent({ command: 'wiki-read', projectDir: tempDir, json: true, pageId: 'missing' }, io.io, {
createRuntime: async () =>
runtime({
ports: { knowledge: { read: vi.fn(async () => null) } },
}) as never,
}),
).resolves.toBe(1);
expect(JSON.parse(io.stderr())).toMatchObject({
ok: false,
error: { message: expect.stringContaining('missing') },
});
});
});

214
packages/cli/src/agent.ts Normal file
View file

@ -0,0 +1,214 @@
import { readFile } from 'node:fs/promises';
import type { KloCliIo } from './cli-runtime.js';
import {
createKloAgentRuntime,
parseAgentMaxRows,
readAgentJsonFile,
writeAgentJson,
writeAgentJsonError,
type KloAgentRuntime,
type KloAgentRuntimeDeps,
} from './agent-runtime.js';
import {
isMissingProjectConfigError,
missingConnectionSlSearchReadiness,
missingProjectSlSearchReadiness,
noConnectionsSlSearchReadiness,
noIndexedSourcesSlSearchReadiness,
type KloAgentSlSearchReadinessDetail,
} from './agent-search-readiness.js';
import { readKloSetupStatus, type KloSetupStatus } from './setup.js';
export type KloAgentArgs =
| { command: 'tools'; projectDir: string; json: true }
| { command: 'context'; projectDir: string; json: true }
| { command: 'sl-list'; projectDir: string; json: true; connectionId?: string; query?: string }
| { command: 'sl-read'; projectDir: string; json: true; connectionId?: string; sourceName: string }
| {
command: 'sl-query';
projectDir: string;
json: true;
connectionId: string;
queryFile: string;
execute: boolean;
maxRows?: number;
}
| { command: 'wiki-search'; projectDir: string; json: true; query: string; limit: number }
| { command: 'wiki-read'; projectDir: string; json: true; pageId: string }
| { command: 'sql-execute'; projectDir: string; json: true; connectionId: string; sqlFile: string; maxRows?: number };
export interface KloAgentDeps extends KloAgentRuntimeDeps {
createRuntime?: (options: {
projectDir: string;
enableSemanticCompute: boolean;
enableQueryExecution: boolean;
}) => Promise<KloAgentRuntime>;
readSetupStatus?: (
projectDir: string,
) => Promise<KloSetupStatus | { project: { path?: string; ready: boolean }; agents: unknown[] }>;
}
const AGENT_TOOLS = [
{ name: 'context', command: 'klo agent context --json' },
{ name: 'sl.list', command: 'klo agent sl list --json [--connection-id <id>] [--query <text>]' },
{ name: 'sl.read', command: 'klo agent sl read <sourceName> --json [--connection-id <id>]' },
{
name: 'sl.query',
command: 'klo agent sl query --json --connection-id <id> --query-file <path> --execute --max-rows 100',
},
{ name: 'wiki.search', command: 'klo agent wiki search <query> --json [--limit 10]' },
{ name: 'wiki.read', command: 'klo agent wiki read <pageId> --json' },
{
name: 'sql.execute',
command: 'klo agent sql execute --json --connection-id <id> --sql-file <path> --max-rows 100',
},
] as const;
function writeAgentSlSearchReadinessError(io: KloCliIo, detail: KloAgentSlSearchReadinessDetail): void {
writeAgentJsonError(io, detail.message, { code: detail.code, nextSteps: detail.nextSteps });
}
async function runtimeFor(args: KloAgentArgs, deps: KloAgentDeps): Promise<KloAgentRuntime> {
const needsSemanticCompute = args.command === 'sl-query';
const needsQueryExecution = args.command === 'sql-execute' || (args.command === 'sl-query' && args.execute);
return deps.createRuntime
? deps.createRuntime({
projectDir: args.projectDir,
enableSemanticCompute: needsSemanticCompute,
enableQueryExecution: needsQueryExecution,
})
: createKloAgentRuntime(
{
projectDir: args.projectDir,
enableSemanticCompute: needsSemanticCompute,
enableQueryExecution: needsQueryExecution,
},
deps,
);
}
function connectionIdForSource(runtime: KloAgentRuntime, requested: string | undefined): string {
if (requested) return requested;
const ids = Object.keys(runtime.project.config.connections ?? {});
if (ids.length === 1) return ids[0] as string;
throw new Error('Use --connection-id when the project has zero or multiple connections.');
}
export async function runKloAgent(args: KloAgentArgs, io: KloCliIo, deps: KloAgentDeps = {}): Promise<number> {
try {
if (args.command === 'tools') {
writeAgentJson(io, { projectDir: args.projectDir, tools: AGENT_TOOLS });
return 0;
}
const runtime = await runtimeFor(args, deps);
if (args.command === 'context') {
const [status, connections, semanticLayer] = await Promise.all([
(deps.readSetupStatus ?? readKloSetupStatus)(args.projectDir),
runtime.ports.connections?.list() ?? [],
runtime.ports.semanticLayer?.listSources({}) ?? { sources: [], totalSources: 0 },
]);
writeAgentJson(io, { projectDir: args.projectDir, status, connections, semanticLayer, tools: AGENT_TOOLS });
return 0;
}
if (args.command === 'sl-list') {
const semanticLayer = runtime.ports.semanticLayer;
if (!semanticLayer) throw new Error('Semantic-layer tools are not available for this project.');
if (args.query) {
const connectionIds = Object.keys(runtime.project.config.connections ?? {});
if (args.connectionId && !runtime.project.config.connections[args.connectionId]) {
writeAgentSlSearchReadinessError(
io,
missingConnectionSlSearchReadiness(args.projectDir, args.connectionId, args.query),
);
return 1;
}
if (connectionIds.length === 0) {
writeAgentSlSearchReadinessError(io, noConnectionsSlSearchReadiness(args.projectDir, args.query));
return 1;
}
}
const listed = await semanticLayer.listSources({ connectionId: args.connectionId, query: args.query });
if (args.query && listed.sources.length === 0) {
const allSources = await semanticLayer.listSources({ connectionId: args.connectionId });
if (allSources.totalSources === 0) {
writeAgentSlSearchReadinessError(io, noIndexedSourcesSlSearchReadiness(args.projectDir, args.query));
return 1;
}
}
writeAgentJson(io, listed);
return 0;
}
if (args.command === 'sl-read') {
const semanticLayer = runtime.ports.semanticLayer;
if (!semanticLayer) throw new Error('Semantic-layer tools are not available for this project.');
const source = await semanticLayer.readSource({
connectionId: connectionIdForSource(runtime, args.connectionId),
sourceName: args.sourceName,
});
if (!source) throw new Error(`Semantic-layer source "${args.sourceName}" was not found.`);
writeAgentJson(io, source);
return 0;
}
if (args.command === 'sl-query') {
const semanticLayer = runtime.ports.semanticLayer;
if (!semanticLayer) throw new Error('Semantic-layer tools are not available for this project.');
const query = await readAgentJsonFile(args.queryFile);
const maxRows = args.execute ? parseAgentMaxRows(args.maxRows) : args.maxRows;
writeAgentJson(
io,
await semanticLayer.query({
connectionId: args.connectionId,
query: { ...query, ...(maxRows !== undefined ? { limit: maxRows } : {}) } as never,
}),
);
return 0;
}
if (args.command === 'wiki-search') {
const knowledge = runtime.ports.knowledge;
if (!knowledge) throw new Error('Wiki tools are not available for this project.');
writeAgentJson(io, await knowledge.search({ userId: 'agent', query: args.query, limit: args.limit }));
return 0;
}
if (args.command === 'wiki-read') {
const knowledge = runtime.ports.knowledge;
if (!knowledge) throw new Error('Wiki tools are not available for this project.');
const page = await knowledge.read({ userId: 'agent', key: args.pageId });
if (!page) throw new Error(`Wiki page "${args.pageId}" was not found.`);
writeAgentJson(io, page);
return 0;
}
const queryExecutor = runtime.queryExecutor;
if (!queryExecutor) throw new Error('SQL execution is not available for this project.');
const connection = runtime.project.config.connections[args.connectionId];
if (!connection) throw new Error(`Connection "${args.connectionId}" was not found.`);
const maxRows = parseAgentMaxRows(args.maxRows);
writeAgentJson(
io,
await queryExecutor.execute({
connectionId: args.connectionId,
projectDir: runtime.project.projectDir,
connection,
sql: await readFile(args.sqlFile, 'utf-8'),
maxRows,
}),
);
return 0;
} catch (error) {
if (args.command === 'sl-list' && args.query && isMissingProjectConfigError(error)) {
writeAgentSlSearchReadinessError(io, missingProjectSlSearchReadiness(args.projectDir, args.query));
return 1;
}
writeAgentJsonError(io, error instanceof Error ? error.message : String(error));
return 1;
}
}

9
packages/cli/src/bin.ts Normal file
View file

@ -0,0 +1,9 @@
#!/usr/bin/env node
import { installStartupProfileReporter, profileMark, profileSpan } from './startup-profile.js';
installStartupProfileReporter();
profileMark('bin:entry');
const { runKloCli } = await profileSpan('import ./cli-runtime.js', () => import('./cli-runtime.js'));
profileMark('bin:runKloCli');
process.exitCode = await runKloCli(process.argv.slice(2));

11
packages/cli/src/clack.ts Normal file
View file

@ -0,0 +1,11 @@
import { spinner } from '@clack/prompts';
export interface KloCliSpinner {
start(message: string): void;
stop(message: string): void;
error(message: string): void;
}
export function createClackSpinner(): KloCliSpinner {
return spinner();
}

View file

@ -0,0 +1,268 @@
import { Command, InvalidArgumentError } from '@commander-js/extra-typings';
import type { KloCliDeps, KloCliIo, KloCliPackageInfo } from './cli-runtime.js';
import { registerAgentCommands } from './commands/agent-commands.js';
import { registerConnectionCommands } from './commands/connection-commands.js';
import { registerWikiCommands } from './commands/knowledge-commands.js';
import { registerPublicIngestCommands } from './commands/public-ingest-commands.js';
import { registerServeCommands } from './commands/serve-commands.js';
import { registerSetupCommands } from './commands/setup-commands.js';
import { registerSlCommands } from './commands/sl-commands.js';
import { registerStatusCommands } from './commands/status-commands.js';
import { registerDevCommands } from './dev.js';
import { findNearestKloProjectDir, resolveKloProjectDir } from './project-resolver.js';
import { profileMark, profileSpan } from './startup-profile.js';
profileMark('module:cli-program');
export interface KloCliCommandContext {
io: KloCliIo;
deps: KloCliDeps;
setExitCode: (code: number) => void;
runInit: (args: { projectDir: string; projectName?: string; force: boolean }, io: KloCliIo) => Promise<number>;
writeDebug?: (command: string, commandContext: CommandWithGlobalOptions) => void;
}
export interface OutputModeOptions {
plain?: boolean;
json?: boolean;
viz?: boolean;
input?: boolean;
}
interface KloCommanderProgramOptions {
runInit: (args: { projectDir: string; projectName?: string; force: boolean }, io: KloCliIo) => Promise<number>;
}
type CommanderExitLike = { exitCode: number; code: string; message: string };
interface KloGlobalOptionValues {
projectDir?: string;
debug?: boolean;
}
export interface CommandWithGlobalOptions {
opts: () => object;
optsWithGlobals?: () => object;
}
function isCommanderExit(error: unknown): error is CommanderExitLike {
return (
typeof error === 'object' &&
error !== null &&
'exitCode' in error &&
typeof (error as { exitCode: unknown }).exitCode === 'number' &&
'code' in error &&
typeof (error as { code: unknown }).code === 'string'
);
}
export function collectOption(value: string, previous: string[] = []): string[] {
return [...previous, value];
}
export function parsePositiveIntegerOption(value: string): number {
const parsed = Number(value);
if (!Number.isInteger(parsed) || parsed < 1) {
throw new InvalidArgumentError('must be a positive integer');
}
return parsed;
}
export function parseNonNegativeIntegerOption(value: string): number {
const parsed = Number(value);
if (!Number.isInteger(parsed) || parsed < 0) {
throw new InvalidArgumentError('must be a non-negative integer');
}
return parsed;
}
export function parseBooleanStringOption(value: string): boolean {
if (value === 'true') {
return true;
}
if (value === 'false') {
return false;
}
throw new InvalidArgumentError('must be true or false');
}
export function parseSafeConnectionIdOption(value: string): string {
if (!/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(value)) {
throw new InvalidArgumentError(`Unsafe connection id: ${value}`);
}
return value;
}
export function parseNonEmptyAssignmentOption(value: string): { key: string; value: string } {
const separatorIndex = value.indexOf('=');
if (separatorIndex <= 0 || separatorIndex === value.length - 1) {
throw new InvalidArgumentError('must be a non-empty <key>=<value> assignment');
}
return {
key: value.slice(0, separatorIndex),
value: value.slice(separatorIndex + 1),
};
}
function optionsWithGlobals(command: CommandWithGlobalOptions): KloGlobalOptionValues {
const options = command.optsWithGlobals ? command.optsWithGlobals() : command.opts();
const values = options as { projectDir?: unknown; debug?: unknown };
return {
projectDir: typeof values.projectDir === 'string' ? values.projectDir : undefined,
debug: typeof values.debug === 'boolean' ? values.debug : undefined,
};
}
export function resolveCommandProjectDir(command: CommandWithGlobalOptions): string {
return resolveKloProjectDir({ explicitProjectDir: optionsWithGlobals(command).projectDir });
}
export function resolveCommandProjectDirOverride(command: CommandWithGlobalOptions): string | undefined {
return optionsWithGlobals(command).projectDir ?? process.env.KLO_PROJECT_DIR;
}
function createBaseProgram(info: KloCliPackageInfo, io: KloCliIo): Command {
return new Command()
.name('klo')
.description('Standalone KLO developer CLI')
.option('--project-dir <path>', 'KLO project directory (default: KLO_PROJECT_DIR, nearest klo.yaml, or cwd)')
.option('--debug', 'Enable diagnostic logging to stderr')
.version(`${info.name} ${info.version}`, '-v, --version', 'Show CLI version')
.helpOption('-h, --help', 'Show this help text')
.configureHelp({ showGlobalOptions: true })
.addHelpText(
'after',
'\nAdvanced:\n klo dev Low-level diagnostics, scans, adapter commands, and mapping tools.\n',
)
.showHelpAfterError()
.exitOverride()
.configureOutput({
writeOut: (chunk) => io.stdout.write(chunk),
writeErr: (chunk) => io.stderr.write(chunk),
outputError: (chunk, write) => write(chunk),
});
}
function writeDebug(io: KloCliIo, commandContext: CommandWithGlobalOptions, command: string): void {
const global = optionsWithGlobals(commandContext);
if (global.debug !== true) {
return;
}
io.stderr.write(`[debug] projectDir=${resolveCommandProjectDir(commandContext)}\n`);
io.stderr.write(`[debug] dispatch=${command}\n`);
}
function formatCliError(error: unknown): string {
return error instanceof Error ? error.message : String(error);
}
async function runBareInteractiveCommand(
program: Command,
io: KloCliIo,
context: KloCliCommandContext,
): Promise<number> {
const nearestProjectDir = findNearestKloProjectDir(process.cwd());
const envProjectDir = process.env.KLO_PROJECT_DIR;
const runner = context.deps.setup ?? (await import('./setup.js')).runKloSetup;
if (!nearestProjectDir && !envProjectDir) {
return await runner(
{
command: 'run',
projectDir: resolveKloProjectDir(),
mode: 'auto',
agents: false,
agentScope: 'project',
agentInstallMode: 'cli',
skipAgents: false,
inputMode: 'auto',
yes: false,
skipLlm: false,
skipEmbeddings: false,
databaseSchemas: [],
skipDatabases: false,
skipSources: false,
},
io,
);
}
program.outputHelp();
return 0;
}
export async function runCommanderKloCli(
argv: string[],
io: KloCliIo,
deps: KloCliDeps,
info: KloCliPackageInfo,
options: KloCommanderProgramOptions,
): Promise<number> {
profileMark('commander:entry');
let exitCode = 0;
const program = createBaseProgram(info, io);
profileMark('commander:base-program');
const context: KloCliCommandContext = {
io,
deps,
setExitCode: (code: number) => {
exitCode = code;
},
runInit: options.runInit,
writeDebug: (command: string, commandContext: CommandWithGlobalOptions) => {
writeDebug(io, commandContext, command);
},
};
registerSetupCommands(program, context);
profileMark('commander:register-setup');
registerConnectionCommands(program, context);
profileMark('commander:register-connection');
registerPublicIngestCommands(program, context);
profileMark('commander:register-public-ingest');
registerWikiCommands(program, context);
profileMark('commander:register-wiki');
registerSlCommands(program, context);
profileMark('commander:register-sl');
registerServeCommands(program, context);
profileMark('commander:register-serve');
registerStatusCommands(program, context);
profileMark('commander:register-status');
registerAgentCommands(program, context);
profileMark('commander:register-agent');
registerDevCommands(program, context);
profileMark('commander:register-dev');
if (argv.length === 0) {
if (io.stdout.isTTY === true) {
try {
return await runBareInteractiveCommand(program, io, context);
} catch (error) {
io.stderr.write(`${formatCliError(error)}\n`);
return 1;
}
}
program.outputHelp();
return 0;
}
try {
await profileSpan('commander:parseAsync', () => program.parseAsync(argv, { from: 'user' }));
} catch (error) {
if (isCommanderExit(error)) {
return error.exitCode === 0 ? 0 : 1;
}
io.stderr.write(`${formatCliError(error)}\n`);
return 1;
}
return exitCode;
}

View file

@ -0,0 +1,89 @@
import type { KloConnectionMetabaseSetupArgs } from './commands/connection-metabase-setup.js';
import type { KloConnectionNotionArgs } from './commands/connection-notion.js';
import type { KloAgentArgs } from './agent.js';
import type { KloConnectionArgs } from './connection.js';
import type { KloDemoArgs } from './demo.js';
import type { KloDoctorArgs } from './doctor.js';
import type { KloIngestArgs } from './ingest.js';
import type { KloKnowledgeArgs } from './knowledge.js';
import type { KloPublicIngestArgs } from './public-ingest.js';
import type { KloScanArgs } from './scan.js';
import type { KloServeArgs } from './serve.js';
import type { KloSetupArgs } from './setup.js';
import type { KloSlArgs } from './sl.js';
import { profileMark, profileSpan } from './startup-profile.js';
profileMark('module:cli-runtime');
export interface KloCliPackageInfo {
name: '@klo/cli';
version: '0.0.0-private';
contextPackageName: '@klo/context';
}
export interface KloCliIo {
stdout: { isTTY?: boolean; write(chunk: string): void };
stderr: { write(chunk: string): void };
}
export interface KloCliDeps {
serveStdio?: (args: KloServeArgs) => Promise<number>;
setup?: (args: KloSetupArgs, io: KloCliIo) => Promise<number>;
agent?: (args: KloAgentArgs, io: KloCliIo) => Promise<number>;
connection?: (args: KloConnectionArgs, io: KloCliIo) => Promise<number>;
connectionNotion?: (args: KloConnectionNotionArgs, io: KloCliIo) => Promise<number>;
connectionMetabaseSetup?: (args: KloConnectionMetabaseSetupArgs, io: KloCliIo) => Promise<number>;
demo?: (args: KloDemoArgs, io: KloCliIo) => Promise<number>;
doctor?: (args: KloDoctorArgs, io: KloCliIo) => Promise<number>;
ingest?: (args: KloIngestArgs, io: KloCliIo) => Promise<number>;
publicIngest?: (args: KloPublicIngestArgs, io: KloCliIo) => Promise<number>;
scan?: (args: KloScanArgs, io: KloCliIo) => Promise<number>;
knowledge?: (args: KloKnowledgeArgs, io: KloCliIo) => Promise<number>;
sl?: (args: KloSlArgs, io: KloCliIo) => Promise<number>;
}
export function getKloCliPackageInfo(): KloCliPackageInfo {
return {
name: '@klo/cli',
version: '0.0.0-private',
contextPackageName: '@klo/context',
};
}
async function runInit(
args: { projectDir: string; projectName?: string; force: boolean },
io: KloCliIo,
): Promise<number> {
const { initKloProject } = await import('@klo/context/project');
const result = await initKloProject({
projectDir: args.projectDir,
projectName: args.projectName,
force: args.force,
});
io.stdout.write(`Initialized KLO project at ${result.projectDir}\n`);
io.stdout.write(`Config: ${result.configPath}\n`);
io.stdout.write(`Commit: ${result.commitHash ?? 'none'}\n`);
return 0;
}
export async function runInitForCommander(
args: { projectDir: string; projectName?: string; force: boolean },
io: KloCliIo,
): Promise<number> {
return await runInit(args, io);
}
export async function runKloCli(
argv = process.argv.slice(2),
io: KloCliIo = process,
deps: KloCliDeps = {},
): Promise<number> {
const info = getKloCliPackageInfo();
profileMark('runtime:runKloCli');
const { runCommanderKloCli } = await profileSpan('import ./cli-program.js', () => import('./cli-program.js'));
return await runCommanderKloCli(argv, io, deps, info, {
runInit: runInitForCommander,
});
}

View file

@ -0,0 +1,85 @@
import { z } from 'zod';
const projectDirSchema = z.string().min(1);
const safeConnectionIdSchema = z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/, 'Unsafe connection id');
const stringArraySchema = z.array(z.string());
export const connectionAddCommandSchema = z.object({
command: z.literal('add'),
projectDir: projectDirSchema,
driver: z.string().min(1),
connectionId: safeConnectionIdSchema,
url: z.string().optional(),
schemas: stringArraySchema,
readonly: z.boolean(),
force: z.boolean(),
allowLiteralCredentials: z.boolean(),
notion: z
.object({
authTokenRef: z.string().min(1),
crawlMode: z.enum(['all_accessible', 'selected_roots']),
rootPageIds: stringArraySchema,
rootDatabaseIds: stringArraySchema,
rootDataSourceIds: stringArraySchema,
maxPagesPerRun: z.number().int().positive().optional(),
maxKnowledgeCreatesPerRun: z.number().int().nonnegative().optional(),
maxKnowledgeUpdatesPerRun: z.number().int().nonnegative().optional(),
})
.optional(),
});
export const wikiWriteCommandSchema = z.object({
command: z.literal('write'),
projectDir: projectDirSchema,
key: z.string().min(1),
scope: z.enum(['GLOBAL', 'USER']),
userId: z.string().min(1),
summary: z.string().min(1),
content: z.string().min(1),
tags: stringArraySchema,
refs: stringArraySchema,
slRefs: stringArraySchema,
});
const orderBySchema = z.union([
z.string().min(1),
z.object({
field: z.string().min(1),
direction: z.enum(['asc', 'desc']).optional(),
}),
]);
export const slQueryCommandSchema = z.object({
command: z.literal('query'),
projectDir: projectDirSchema,
connectionId: z.string().min(1).optional(),
query: z.object({
measures: z.array(z.string().min(1)).min(1),
dimensions: stringArraySchema,
filters: stringArraySchema.optional(),
segments: stringArraySchema.optional(),
order_by: z.array(orderBySchema).optional(),
limit: z.number().int().positive().optional(),
include_empty: z.literal(true).optional(),
}),
format: z.enum(['json', 'sql']),
execute: z.boolean(),
maxRows: z.number().int().positive().optional(),
});
export const publicIngestRunCommandSchema = z.object({
command: z.literal('run'),
projectDir: projectDirSchema,
targetConnectionId: safeConnectionIdSchema.optional(),
all: z.boolean(),
json: z.boolean(),
inputMode: z.enum(['auto', 'disabled']),
});
export const publicIngestReadCommandSchema = z.object({
command: z.enum(['status', 'watch']),
projectDir: projectDirSchema,
runId: z.string().min(1).optional(),
json: z.boolean(),
inputMode: z.enum(['auto', 'disabled']),
});

View file

@ -0,0 +1,137 @@
import { Option, type Command } from '@commander-js/extra-typings';
import type { KloAgentArgs } from '../agent.js';
import type { KloCliCommandContext } from '../cli-program.js';
import { parsePositiveIntegerOption, resolveCommandProjectDir } from '../cli-program.js';
async function runAgent(context: KloCliCommandContext, args: KloAgentArgs): Promise<void> {
const runner = context.deps.agent ?? (await import('../agent.js')).runKloAgent;
context.setExitCode(await runner(args, context.io));
}
function jsonOption(): Option {
return new Option('--json', 'Print JSON output').makeOptionMandatory();
}
export function registerAgentCommands(program: Command, context: KloCliCommandContext): void {
const agent = program
.command('agent', { hidden: true })
.description('Machine-readable KLO commands for coding agents')
.showHelpAfterError();
agent.hook('preAction', (_thisCommand, actionCommand) => {
context.writeDebug?.('agent', actionCommand);
});
agent
.command('tools')
.description('Print available agent-facing KLO tools')
.addOption(jsonOption())
.action(async (_options, command) => {
await runAgent(context, { command: 'tools', projectDir: resolveCommandProjectDir(command), json: true });
});
agent
.command('context')
.description('Print project context for agent planning')
.addOption(jsonOption())
.action(async (_options, command) => {
await runAgent(context, { command: 'context', projectDir: resolveCommandProjectDir(command), json: true });
});
const sl = agent.command('sl').description('Semantic-layer agent commands');
sl.command('list')
.description('List semantic-layer sources')
.addOption(jsonOption())
.option('--connection-id <id>', 'Filter by connection id')
.option('--query <text>', 'Search source names and descriptions')
.action(async (options: { connectionId?: string; query?: string }, command) => {
await runAgent(context, {
command: 'sl-list',
projectDir: resolveCommandProjectDir(command),
json: true,
...(options.connectionId ? { connectionId: options.connectionId } : {}),
...(options.query ? { query: options.query } : {}),
});
});
sl.command('read')
.description('Read one semantic-layer source')
.argument('<sourceName>')
.addOption(jsonOption())
.option('--connection-id <id>', 'Connection id containing the source')
.action(async (sourceName: string, options: { connectionId?: string }, command) => {
await runAgent(context, {
command: 'sl-read',
projectDir: resolveCommandProjectDir(command),
json: true,
sourceName,
...(options.connectionId ? { connectionId: options.connectionId } : {}),
});
});
sl.command('query')
.description('Run a semantic-layer query JSON file')
.addOption(jsonOption())
.requiredOption('--connection-id <id>', 'Connection id for execution')
.requiredOption('--query-file <path>', 'JSON semantic-layer query file')
.option('--execute', 'Execute the compiled query against the connection', false)
.option('--max-rows <number>', 'Maximum rows to return when executing', parsePositiveIntegerOption)
.action(
async (
options: { connectionId: string; queryFile: string; execute: boolean; maxRows?: number },
command,
) => {
await runAgent(context, {
command: 'sl-query',
projectDir: resolveCommandProjectDir(command),
json: true,
connectionId: options.connectionId,
queryFile: options.queryFile,
execute: options.execute,
...(options.maxRows !== undefined ? { maxRows: options.maxRows } : {}),
});
},
);
const wiki = agent.command('wiki').description('KLO wiki agent commands');
wiki
.command('search')
.description('Search KLO wiki pages')
.argument('<query>')
.addOption(jsonOption())
.option('--limit <number>', 'Maximum search results', parsePositiveIntegerOption, 10)
.action(async (query: string, options: { limit: number }, command) => {
await runAgent(context, {
command: 'wiki-search',
projectDir: resolveCommandProjectDir(command),
json: true,
query,
limit: options.limit,
});
});
wiki
.command('read')
.description('Read one KLO wiki page')
.argument('<pageId>')
.addOption(jsonOption())
.action(async (pageId: string, _options, command) => {
await runAgent(context, { command: 'wiki-read', projectDir: resolveCommandProjectDir(command), json: true, pageId });
});
const sql = agent.command('sql').description('Safe SQL execution commands');
sql
.command('execute')
.description('Execute read-only SQL with a row limit')
.addOption(jsonOption())
.requiredOption('--connection-id <id>', 'Connection id for execution')
.requiredOption('--sql-file <path>', 'SQL file to execute')
.requiredOption('--max-rows <number>', 'Maximum rows to return', parsePositiveIntegerOption)
.action(async (options: { connectionId: string; sqlFile: string; maxRows: number }, command) => {
await runAgent(context, {
command: 'sql-execute',
projectDir: resolveCommandProjectDir(command),
json: true,
connectionId: options.connectionId,
sqlFile: options.sqlFile,
maxRows: options.maxRows,
});
});
}

View file

@ -0,0 +1,47 @@
import type { CommandUnknownOpts } from '@commander-js/extra-typings';
import type { KloCliCommandContext } from '../cli-program.js';
import { completeCommanderInput, installZshCompletion, zshCompletionScript } from '../completion.js';
export function registerCompletionCommands(
program: CommandUnknownOpts,
context: KloCliCommandContext,
completionRoot: CommandUnknownOpts = program,
): void {
program
.command('completion')
.description('Generate shell completion scripts')
.command('zsh')
.description('Generate zsh completion script')
.option('--install', 'Install zsh completion into ~/.zfunc and update ~/.zshrc', false)
.action(async (options: { install?: boolean }) => {
if (options.install === true) {
const result = await installZshCompletion();
context.io.stdout.write(`Installed zsh completion: ${result.completionPath}\n`);
context.io.stdout.write(`Updated zsh config: ${result.zshrcPath}\n`);
context.io.stdout.write('Restart your shell or run: source ~/.zshrc\n');
context.setExitCode(0);
return;
}
context.io.stdout.write(zshCompletionScript());
context.setExitCode(0);
});
program
.command('__complete', { hidden: true })
.description('Internal shell completion endpoint')
.requiredOption('--shell <shell>', 'Shell requesting completions')
.requiredOption('--position <position>', 'Current shell word position', (value) => Number(value))
.argument('[words...]', 'Current shell words')
.allowUnknownOption()
.allowExcessArguments()
.action((words: string[], options: { shell: string; position: number }) => {
if (options.shell !== 'zsh') {
context.setExitCode(1);
return;
}
for (const completion of completeCommanderInput(completionRoot, { position: options.position, words })) {
context.io.stdout.write(`${completion}\n`);
}
context.setExitCode(0);
});
}

View file

@ -0,0 +1,346 @@
import { type Command, InvalidArgumentError, Option } from '@commander-js/extra-typings';
import {
collectOption,
type KloCliCommandContext,
parseBooleanStringOption,
parseNonEmptyAssignmentOption,
parseNonNegativeIntegerOption,
parsePositiveIntegerOption,
parseSafeConnectionIdOption,
resolveCommandProjectDir,
} from '../cli-program.js';
import { connectionAddCommandSchema } from '../command-schemas.js';
import type { KloConnectionArgs } from '../connection.js';
import { profileMark } from '../startup-profile.js';
import type { KloConnectionMappingArgs } from './connection-mapping.js';
import { registerConnectionMetabaseCommands } from './connection-metabase-commands.js';
import { registerConnectionNotionCommands } from './connection-notion-commands.js';
profileMark('module:commands/connection-commands');
const CRAWL_MODE_CHOICES = ['all_accessible', 'selected_roots'] as const;
const SYNC_MODE_CHOICES = ['ALL', 'ONLY', 'EXCEPT'] as const;
function parseCsvIds(value: string): number[] {
return value
.split(',')
.filter(Boolean)
.map((item) => parsePositiveIntegerOption(item));
}
function parseCsvStrings(value: string): string[] {
return value
.split(',')
.map((item) => item.trim())
.filter(Boolean);
}
function parseMappingFieldOption(value: string): 'databaseMappings' | 'connectionMappings' {
if (value === 'databaseMappings' || value === 'connectionMappings') {
return value;
}
throw new InvalidArgumentError('must be databaseMappings or connectionMappings');
}
async function runConnectionArgs(context: KloCliCommandContext, args: KloConnectionArgs): Promise<void> {
const runner = context.deps.connection ?? (await import('../connection.js')).runKloConnection;
context.setExitCode(await runner(args, context.io));
}
async function runMappingArgs(context: KloCliCommandContext, args: KloConnectionMappingArgs): Promise<void> {
const { runKloConnectionMapping } = await import('./connection-mapping.js');
context.setExitCode(await runKloConnectionMapping(args, context.io));
}
export function registerConnectionCommands(program: Command, context: KloCliCommandContext, commandName = 'connection'): void {
const connection = program
.command(commandName)
.description('Add, list, test, and map data sources')
.showHelpAfterError()
.addHelpText(
'after',
'\nProject directory defaults to KLO_PROJECT_DIR when set, otherwise the nearest klo.yaml or current working directory.\n',
);
connection.hook('preAction', (_thisCommand, actionCommand) => {
context.writeDebug?.(commandName, actionCommand);
});
connection
.command('list')
.description('List configured connections')
.action(async (_options: unknown, command) => {
await runConnectionArgs(context, { command: 'list', projectDir: resolveCommandProjectDir(command) });
});
connection
.command('test')
.description('Test a configured connection')
.argument('<connectionId>', 'KLO connection id')
.action(async (connectionId: string, _options: unknown, command) => {
await runConnectionArgs(context, {
command: 'test',
projectDir: resolveCommandProjectDir(command),
connectionId,
});
});
connection
.command('add')
.description('Add or replace a configured connection')
.argument('<driver>', 'Connection driver')
.argument('<connectionId>', 'KLO connection id')
.option('--url <url>', 'Connection URL, env:NAME, or file:/path reference')
.option('--schema <schema>', 'Schema to include; repeatable', collectOption, [])
.option('--readonly', 'Mark the connection as read-only', false)
.option('--force', 'Replace an existing connection', false)
.option('--allow-literal-credentials', 'Allow writing a literal credential URL to klo.yaml', false)
.addOption(new Option('--token-env <name>', 'Environment variable containing Notion auth token').conflicts('tokenFile'))
.addOption(new Option('--token-file <path>', 'File containing Notion auth token').conflicts('tokenEnv'))
.addOption(
new Option('--crawl-mode <mode>', 'Notion crawl mode: all_accessible or selected_roots')
.choices(CRAWL_MODE_CHOICES)
.default('selected_roots'),
)
.option('--root-page-id <id>', 'Root page to crawl; repeatable', collectOption, [])
.option('--root-database-id <id>', 'Root database to crawl; repeatable', collectOption, [])
.option('--root-data-source-id <id>', 'Root data source to crawl; repeatable', collectOption, [])
.option('--max-pages <n>', 'Maximum pages per run', parsePositiveIntegerOption)
.option('--max-knowledge-creates <n>', 'Maximum knowledge creates per run', parseNonNegativeIntegerOption)
.option('--max-knowledge-updates <n>', 'Maximum knowledge updates per run', parseNonNegativeIntegerOption)
.action(async (driver: string, connectionId: string, options, command) => {
const notion =
driver === 'notion'
? {
authTokenRef: options.tokenEnv
? `env:${options.tokenEnv}`
: options.tokenFile
? `file:${options.tokenFile}`
: '',
crawlMode: options.crawlMode,
rootPageIds: options.rootPageId,
rootDatabaseIds: options.rootDatabaseId,
rootDataSourceIds: options.rootDataSourceId,
maxPagesPerRun: options.maxPages,
maxKnowledgeCreatesPerRun: options.maxKnowledgeCreates,
maxKnowledgeUpdatesPerRun: options.maxKnowledgeUpdates,
}
: undefined;
if (driver === 'notion' && !notion?.authTokenRef) {
throw new Error('connection add notion requires --token-env NAME or --token-file PATH');
}
if (
driver === 'notion' &&
notion?.crawlMode === 'selected_roots' &&
notion.rootPageIds.length + notion.rootDatabaseIds.length + notion.rootDataSourceIds.length === 0
) {
throw new Error('connection add notion selected_roots requires at least one root id');
}
const args = connectionAddCommandSchema.parse({
command: 'add',
projectDir: resolveCommandProjectDir(command),
driver,
connectionId,
url: options.url,
schemas: options.schema.filter(Boolean),
readonly: options.readonly === true,
force: options.force === true,
allowLiteralCredentials: options.allowLiteralCredentials === true,
notion,
});
await runConnectionArgs(context, args);
});
connection
.command('remove')
.description('Remove a configured connection from klo.yaml')
.argument('<connectionId>', 'KLO connection id')
.option('--force', 'Remove without prompting', false)
.option('--no-input', 'Disable interactive terminal input')
.action(async (connectionId: string, options: { force?: boolean; input?: boolean }, command) => {
await runConnectionArgs(context, {
command: 'remove',
projectDir: resolveCommandProjectDir(command),
connectionId,
force: options.force === true,
...(options.input === false ? { inputMode: 'disabled' } : {}),
});
});
connection
.command('map')
.description('Refresh and validate BI-to-warehouse mappings')
.argument('<sourceConnectionId>', 'Source BI connection id')
.option('--json', 'Print JSON output', false)
.action(async (sourceConnectionId: string, options: { json?: boolean }, command) => {
await runConnectionArgs(context, {
command: 'map',
projectDir: resolveCommandProjectDir(command),
sourceConnectionId,
json: options.json === true,
});
});
registerConnectionMappingCommands(connection, context);
registerConnectionMetabaseCommands(connection, context);
registerConnectionNotionCommands(connection, context);
}
export function registerConnectionMappingCommands(connection: Command, context: KloCliCommandContext): void {
const mapping = connection
.command('mapping')
.description('Manage Metabase warehouse mappings')
.showHelpAfterError()
.addHelpText(
'after',
'\nProject directory defaults to KLO_PROJECT_DIR when set, otherwise the current working directory.\n',
);
mapping
.command('list')
.description('List Metabase database mappings')
.argument('<connectionId>', 'Metabase connection id')
.option('--json', 'Print JSON output where supported', false)
.action(async (connectionId: string, options: { json?: boolean }, command) => {
await runMappingArgs(context, {
command: 'list',
projectDir: resolveCommandProjectDir(command),
connectionId,
json: options.json === true,
});
});
mapping
.command('set')
.description('Set a Metabase or Looker warehouse mapping')
.argument('<connectionId>', 'Source connection id', parseSafeConnectionIdOption)
.argument('<field>', 'Mapping field', parseMappingFieldOption)
.argument('<assignment>', 'Mapping assignment such as 1=prod-warehouse', parseNonEmptyAssignmentOption)
.action(
async (
connectionId: string,
field: 'databaseMappings' | 'connectionMappings',
assignment: { key: string; value: string },
_options: unknown,
command,
) => {
await runMappingArgs(context, {
command: 'set',
projectDir: resolveCommandProjectDir(command),
connectionId,
field,
key: assignment.key,
value: assignment.value,
});
},
);
mapping
.command('apply-bulk')
.description('Apply mappings from JSON')
.argument('<connectionId>', 'Metabase connection id')
.requiredOption('--file <path>', 'JSON mapping file')
.action(async (connectionId: string, options: { file: string }, command) => {
await runMappingArgs(context, {
command: 'apply-bulk',
projectDir: resolveCommandProjectDir(command),
connectionId,
filePath: options.file,
});
});
mapping
.command('set-sync-enabled')
.description('Enable or disable sync for one Metabase database')
.argument('<connectionId>', 'Metabase connection id')
.argument('<metabaseDatabaseId>', 'Metabase database id', parsePositiveIntegerOption)
.requiredOption('--enabled <value>', 'true or false', parseBooleanStringOption)
.action(
async (connectionId: string, metabaseDatabaseId: number, options: { enabled: boolean }, command) => {
await runMappingArgs(context, {
command: 'set-sync-enabled',
projectDir: resolveCommandProjectDir(command),
connectionId,
metabaseDatabaseId,
enabled: options.enabled,
});
},
);
const syncState = mapping.command('sync-state').description('Manage Metabase sync-state selection');
syncState
.command('get')
.description('Read sync-state selection')
.argument('<connectionId>', 'Metabase connection id')
.option('--json', 'Print JSON output where supported', false)
.action(async (connectionId: string, options: { json?: boolean }, command) => {
await runMappingArgs(context, {
command: 'sync-state-get',
projectDir: resolveCommandProjectDir(command),
connectionId,
json: options.json === true,
});
});
syncState
.command('set')
.description('Write sync-state selection')
.argument('<connectionId>', 'Metabase connection id')
.addOption(new Option('--mode <mode>', 'ALL, ONLY, or EXCEPT').choices(SYNC_MODE_CHOICES).makeOptionMandatory())
.option('--collections <ids>', 'Comma-separated collection ids', parseCsvIds, [])
.option('--items <ids>', 'Comma-separated item ids', parseCsvIds, [])
.option('--tag-names <names>', 'Comma-separated tag names', parseCsvStrings, [])
.action(async (connectionId: string, options, command) => {
await runMappingArgs(context, {
command: 'sync-state-set',
projectDir: resolveCommandProjectDir(command),
connectionId,
syncMode: options.mode,
collectionIds: options.collections,
itemIds: options.items,
tagNames: options.tagNames,
});
});
mapping
.command('refresh')
.description('Refresh Metabase database mappings')
.argument('<connectionId>', 'Metabase connection id')
.option('--auto-accept', 'Accept refresh changes without prompting', false)
.action(async (connectionId: string, options: { autoAccept?: boolean }, command) => {
await runMappingArgs(context, {
command: 'refresh',
projectDir: resolveCommandProjectDir(command),
connectionId,
autoAccept: options.autoAccept === true,
});
});
mapping
.command('validate')
.description('Validate Metabase database mappings')
.argument('<connectionId>', 'Metabase connection id')
.action(async (connectionId: string, _options: unknown, command) => {
await runMappingArgs(context, {
command: 'validate',
projectDir: resolveCommandProjectDir(command),
connectionId,
});
});
mapping
.command('clear')
.description('Clear Metabase database mappings')
.argument('<connectionId>', 'Metabase connection id')
.argument('[metabaseDatabaseId]', 'Metabase database id', parsePositiveIntegerOption)
.action(async (connectionId: string, metabaseDatabaseId: number | undefined, _options: unknown, command) => {
await runMappingArgs(context, {
command: 'clear',
projectDir: resolveCommandProjectDir(command),
connectionId,
...(metabaseDatabaseId ? { metabaseDatabaseId } : {}),
});
});
}

View file

@ -0,0 +1,329 @@
import { mkdtemp, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { LocalMetabaseSourceStateReader } from '@klo/context/ingest';
import { initKloProject, loadKloProject, serializeKloProjectConfig } from '@klo/context/project';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { runKloConnectionMapping } from './connection-mapping.js';
function makeIo() {
let stdout = '';
let stderr = '';
return {
io: {
stdout: {
write: (chunk: string) => {
stdout += chunk;
},
},
stderr: {
write: (chunk: string) => {
stderr += chunk;
},
},
},
stdout: () => stdout,
stderr: () => stderr,
};
}
describe('runKloConnectionMapping', () => {
let tempDir: string;
let projectDir: string;
beforeEach(async () => {
tempDir = await mkdtemp(join(tmpdir(), 'klo-cli-metabase-mapping-'));
projectDir = join(tempDir, 'project');
await initKloProject({ projectDir, projectName: 'mapping' });
const project = await loadKloProject({ projectDir });
await project.fileStore.writeFile(
'klo.yaml',
serializeKloProjectConfig({
...project.config,
connections: {
'prod-metabase': {
driver: 'metabase',
api_url: 'https://metabase.example.com',
api_key_ref: 'env:METABASE_API_KEY', // pragma: allowlist secret
},
'prod-warehouse': {
driver: 'postgres',
url: 'env:WAREHOUSE_URL',
readonly: true,
},
},
}),
'klo',
'klo@example.com',
'Seed Metabase mapping test connections',
);
});
async function replaceConnections(connections: Record<string, { driver: string; [key: string]: unknown }>) {
const project = await loadKloProject({ projectDir });
await project.fileStore.writeFile(
'klo.yaml',
serializeKloProjectConfig({
...project.config,
connections,
}),
'klo',
'klo@example.com',
'Replace mapping test connections',
);
}
afterEach(async () => {
await rm(tempDir, { recursive: true, force: true });
});
it('sets, lists, disables, and clears local Metabase mappings', async () => {
const io = makeIo();
await expect(
runKloConnectionMapping(
{
command: 'set',
projectDir,
connectionId: 'prod-metabase',
field: 'databaseMappings',
key: '1',
value: 'prod-warehouse',
},
io.io,
),
).resolves.toBe(0);
const listIo = makeIo();
await expect(
runKloConnectionMapping({ command: 'list', projectDir, connectionId: 'prod-metabase', json: false }, listIo.io),
).resolves.toBe(0);
expect(listIo.stdout()).toContain('1 -> prod-warehouse');
expect(listIo.stdout()).toContain('unhydrated');
await expect(
runKloConnectionMapping(
{
command: 'set-sync-enabled',
projectDir,
connectionId: 'prod-metabase',
metabaseDatabaseId: 1,
enabled: false,
},
makeIo().io,
),
).resolves.toBe(0);
await expect(
runKloConnectionMapping(
{
command: 'clear',
projectDir,
connectionId: 'prod-metabase',
metabaseDatabaseId: 1,
},
makeIo().io,
),
).resolves.toBe(0);
});
it('lists Metabase yaml mapping bootstrap rows before any SQLite command writes', async () => {
const projectDir = await mkdtemp(join(tmpdir(), 'klo-cli-yaml-mapping-'));
await initKloProject({ projectDir, projectName: 'yaml-mapping' });
const project = await loadKloProject({ projectDir });
await project.fileStore.writeFile(
'klo.yaml',
serializeKloProjectConfig({
...project.config,
connections: {
'prod-metabase': {
driver: 'metabase',
mappings: {
databaseMappings: { '1': 'prod-warehouse' },
syncEnabled: { '1': true },
},
},
'prod-warehouse': { driver: 'postgres', url: 'postgresql://readonly@db.test/analytics' },
},
}),
'klo',
'klo@example.com',
'Seed yaml mappings',
);
const io = makeIo();
await expect(
runKloConnectionMapping(
{ command: 'list', projectDir, connectionId: 'prod-metabase', json: false },
io.io,
),
).resolves.toBe(0);
expect(io.stdout()).toContain('1 -> prod-warehouse');
expect(io.stdout()).toContain('source: klo.yaml');
});
it('refreshes Metabase discovery metadata through the injected runtime client', async () => {
const client = {
getDatabases: vi.fn().mockResolvedValue([
{
id: 1,
name: 'Analytics',
engine: 'postgres',
details: { host: 'pg.internal', dbname: 'analytics' },
is_sample: false,
},
]),
cleanup: vi.fn(),
};
const io = makeIo();
await expect(
runKloConnectionMapping(
{
command: 'refresh',
projectDir,
connectionId: 'prod-metabase',
autoAccept: true,
},
io.io,
{
createMetabaseClient: async () => client as never,
},
),
).resolves.toBe(0);
expect(io.stdout()).toContain('Discovery: 1 database');
expect(client.cleanup).toHaveBeenCalledTimes(1);
const store = new LocalMetabaseSourceStateReader({ dbPath: join(projectDir, '.klo', 'db.sqlite') });
await expect(store.listDatabaseMappings('prod-metabase')).resolves.toMatchObject([
{ metabaseDatabaseId: 1, metabaseDatabaseName: 'Analytics', source: 'refresh' },
]);
});
it('sets and lists Looker connection mappings', async () => {
await replaceConnections({
'prod-looker': {
driver: 'looker',
base_url: 'https://looker.example.test',
client_id: 'id',
},
'prod-warehouse': {
driver: 'postgres',
url: 'postgresql://readonly@db.example.test/analytics',
},
});
const io = makeIo();
await expect(
runKloConnectionMapping(
{
command: 'set',
projectDir,
connectionId: 'prod-looker',
field: 'connectionMappings',
key: 'analytics',
value: 'prod-warehouse',
},
io.io,
),
).resolves.toBe(0);
await expect(
runKloConnectionMapping({ command: 'list', projectDir, connectionId: 'prod-looker', json: false }, io.io),
).resolves.toBe(0);
expect(io.stdout()).toContain('analytics -> prod-warehouse');
});
it('keeps driver-specific mapping field validation in the runner', async () => {
await replaceConnections({
'prod-looker': { driver: 'looker', base_url: 'https://looker.example.com' },
warehouse: { driver: 'postgres', url: 'env:WAREHOUSE_URL' },
});
const io = makeIo();
await expect(
runKloConnectionMapping(
{
command: 'set',
projectDir,
connectionId: 'prod-looker',
field: 'databaseMappings',
key: '1',
value: 'warehouse',
},
io.io,
),
).resolves.toBe(1);
expect(io.stderr()).toContain('Looker mapping set requires connectionMappings');
});
it('refreshes Looker mapping metadata and reports drift', async () => {
await replaceConnections({
'prod-looker': {
driver: 'looker',
base_url: 'https://looker.example.test',
client_id: 'id',
},
'prod-warehouse': {
driver: 'postgres',
url: 'postgresql://readonly@db.example.test/analytics',
},
});
const io = makeIo();
await expect(
runKloConnectionMapping(
{ command: 'refresh', projectDir, connectionId: 'prod-looker', autoAccept: true },
io.io,
{
createLookerClient: async () => ({
listLookerConnections: async () => [
{
name: 'analytics',
host: 'db.example.test',
database: 'analytics',
schema: null,
dialect: 'postgres',
},
],
cleanup: async () => {},
}),
},
),
).resolves.toBe(0);
expect(io.stdout()).toContain('Discovery: 1 connection');
expect(io.stdout()).toContain('Unmapped discovered: 1');
});
it('validates Looker mappings through the canonical local warehouse descriptor', async () => {
const projectDir = await mkdtemp(join(tmpdir(), 'klo-cli-descriptor-validation-'));
await initKloProject({ projectDir, projectName: 'descriptor-validation' });
const project = await loadKloProject({ projectDir });
await project.fileStore.writeFile(
'klo.yaml',
serializeKloProjectConfig({
...project.config,
connections: {
'prod-looker': {
driver: 'looker',
mappings: { connectionMappings: { analytics: 'prod-warehouse' } },
},
'prod-warehouse': { driver: 'postgresql', url: 'postgresql://readonly@db.test/analytics' },
},
}),
'klo',
'klo@example.com',
'Seed descriptor validation',
);
const io = makeIo();
await expect(
runKloConnectionMapping({ command: 'validate', projectDir, connectionId: 'prod-looker' }, io.io),
).resolves.toBe(0);
expect(io.stdout()).toContain('Mapping validation passed: prod-looker');
expect(io.stderr()).toBe('');
});
});

View file

@ -0,0 +1,426 @@
import { readFile } from 'node:fs/promises';
import { localConnectionToWarehouseDescriptor } from '@klo/context/connections';
import {
DEFAULT_METABASE_CLIENT_CONFIG,
DefaultLookerConnectionClientFactory,
DefaultMetabaseConnectionClientFactory,
LocalLookerRuntimeStore,
LocalMetabaseSourceStateReader,
computeLookerMappingDrift,
computeMetabaseMappingDrift,
discoverLookerConnections,
discoverMetabaseDatabases,
lookerCredentialsFromLocalConnection,
metabaseRuntimeConfigFromLocalConnection,
seedLocalMappingStateFromKloYaml,
validateLookerMappings,
validateMappingPhysicalMatch,
type LookerMappingClient,
type MetabaseRuntimeClient,
type MetabaseSyncMode,
} from '@klo/context/ingest';
import { type KloLocalProject, kloLocalStateDbPath, loadKloProject } from '@klo/context/project';
import type { KloCliIo } from '../index.js';
import { profileMark } from '../startup-profile.js';
profileMark('module:commands/connection-mapping');
export type KloConnectionMappingArgs =
| { command: 'list'; projectDir: string; connectionId: string; json: boolean }
| {
command: 'set';
projectDir: string;
connectionId: string;
field: 'databaseMappings' | 'connectionMappings';
key: string;
value: string;
}
| { command: 'apply-bulk'; projectDir: string; connectionId: string; filePath: string }
| {
command: 'set-sync-enabled';
projectDir: string;
connectionId: string;
metabaseDatabaseId: number;
enabled: boolean;
}
| { command: 'sync-state-get'; projectDir: string; connectionId: string; json: boolean }
| {
command: 'sync-state-set';
projectDir: string;
connectionId: string;
syncMode: MetabaseSyncMode;
collectionIds: number[];
itemIds: number[];
tagNames: string[];
}
| { command: 'refresh'; projectDir: string; connectionId: string; autoAccept: boolean }
| { command: 'validate'; projectDir: string; connectionId: string }
| { command: 'clear'; projectDir: string; connectionId: string; metabaseDatabaseId?: number; mappingKey?: string };
interface KloConnectionMappingDeps {
createMetabaseClient?: (
project: KloLocalProject,
connectionId: string,
) => Promise<Pick<MetabaseRuntimeClient, 'getDatabases' | 'cleanup'>>;
createLookerClient?: (
project: KloLocalProject,
connectionId: string,
) => Promise<Pick<LookerMappingClient, 'listLookerConnections'> & { cleanup?(): Promise<void> }>;
}
interface MetabaseBulkMappingPayload {
databaseMappings?: Record<string, string | null>;
syncEnabled?: Record<string, boolean>;
syncMode?: MetabaseSyncMode;
selections?: { collections?: number[]; items?: number[] };
defaultTagNames?: string[];
}
function parseId(value: string, label: string): number {
const parsed = Number(value);
if (!Number.isInteger(parsed) || parsed < 1) {
throw new Error(`${label} must be a positive integer`);
}
return parsed;
}
async function createDefaultMetabaseClient(
project: KloLocalProject,
connectionId: string,
): Promise<Pick<MetabaseRuntimeClient, 'getDatabases' | 'cleanup'>> {
const factory = new DefaultMetabaseConnectionClientFactory(
(metabaseConnectionId) =>
metabaseRuntimeConfigFromLocalConnection(metabaseConnectionId, project.config.connections[metabaseConnectionId]),
DEFAULT_METABASE_CLIENT_CONFIG,
);
return factory.createClient(connectionId);
}
async function createDefaultLookerClient(
project: KloLocalProject,
connectionId: string,
): Promise<Pick<LookerMappingClient, 'listLookerConnections'> & { cleanup?(): Promise<void> }> {
const factory = new DefaultLookerConnectionClientFactory({
async resolve(lookerConnectionId) {
return lookerCredentialsFromLocalConnection(lookerConnectionId, project.config.connections[lookerConnectionId]);
},
});
return factory.createClient(connectionId) as unknown as Pick<LookerMappingClient, 'listLookerConnections'> & {
cleanup?(): Promise<void>;
};
}
function isLookerConnection(project: KloLocalProject, connectionId: string): boolean {
return String(project.config.connections[connectionId]?.driver ?? '').toLowerCase() === 'looker';
}
function assertLookerConnection(project: KloLocalProject, connectionId: string): void {
if (!isLookerConnection(project, connectionId)) {
throw new Error(`Connection "${connectionId}" is not a Looker connection`);
}
}
function assertMetabaseConnection(project: KloLocalProject, connectionId: string): void {
const connection = project.config.connections[connectionId];
if (!connection || String(connection.driver).toLowerCase() !== 'metabase') {
throw new Error(`Connection "${connectionId}" is not a Metabase connection`);
}
}
function assertTargetConnection(project: KloLocalProject, connectionId: string): void {
if (!project.config.connections[connectionId]) {
throw new Error(`Target connection "${connectionId}" does not exist`);
}
}
function targetPhysicalInfo(project: KloLocalProject, connectionId: string) {
const descriptor = localConnectionToWarehouseDescriptor(connectionId, project.config.connections[connectionId]);
if (!descriptor) {
return { connection_type: 'UNKNOWN' };
}
return {
connection_type: descriptor.connection_type,
host: descriptor.host ?? null,
database: descriptor.database ?? null,
account: descriptor.account ?? null,
project_id: descriptor.project_id ?? null,
dataset_id: descriptor.dataset_id ?? null,
...descriptor.connection_params,
};
}
function renderMapping(
row: Awaited<ReturnType<LocalMetabaseSourceStateReader['listDatabaseMappings']>>[number],
): string {
const name = row.metabaseDatabaseName ?? 'unhydrated';
const target = row.targetConnectionId ?? '[unmapped]';
return `${row.metabaseDatabaseId} -> ${target} (${name}, sync: ${row.syncEnabled ? 'on' : 'off'}, source: ${
row.source
})`;
}
function renderLookerMapping(row: Awaited<ReturnType<LocalLookerRuntimeStore['listConnectionMappings']>>[number]): string {
const target = row.kloConnectionId ?? '[unmapped]';
const metadata = [row.lookerDialect, row.lookerHost, row.lookerDatabase].filter(Boolean).join(', ');
return `${row.lookerConnectionName} -> ${target}${metadata ? ` (${metadata}, source: ${row.source})` : ` (source: ${row.source})`}`;
}
export async function runKloConnectionMapping(
args: KloConnectionMappingArgs,
io: KloCliIo = process,
deps: KloConnectionMappingDeps = {},
): Promise<number> {
try {
const project = await loadKloProject({ projectDir: args.projectDir });
await seedLocalMappingStateFromKloYaml(project, args.connectionId);
if (isLookerConnection(project, args.connectionId)) {
assertLookerConnection(project, args.connectionId);
const store = new LocalLookerRuntimeStore({ dbPath: kloLocalStateDbPath(project) });
if (args.command === 'list') {
const rows = await store.listConnectionMappings(args.connectionId);
io.stdout.write(args.json ? `${JSON.stringify(rows, null, 2)}\n` : `${rows.map(renderLookerMapping).join('\n')}\n`);
return 0;
}
if (args.command === 'set') {
if (args.field !== 'connectionMappings') {
throw new Error('Looker mapping set requires connectionMappings <lookerConnectionName>=<targetConnectionId>');
}
assertTargetConnection(project, args.value);
await store.upsertConnectionMapping({
lookerConnectionId: args.connectionId,
lookerConnectionName: args.key,
kloConnectionId: args.value,
source: 'cli',
});
io.stdout.write(`Set connectionMappings.${args.key} = ${args.value}\n`);
return 0;
}
if (args.command === 'refresh') {
const client = await (deps.createLookerClient ?? createDefaultLookerClient)(project, args.connectionId);
try {
const discovered = await discoverLookerConnections(client);
const drift = computeLookerMappingDrift({
storedMappings: await store.readMappings(args.connectionId),
discovered,
});
if (args.autoAccept) {
await store.refreshDiscoveredConnections({ lookerConnectionId: args.connectionId, discovered });
}
io.stdout.write(`Discovery: ${discovered.length} ${discovered.length === 1 ? 'connection' : 'connections'}\n`);
io.stdout.write(`Unmapped discovered: ${drift.unmappedDiscovered.length}\n`);
io.stdout.write(`Stale mappings: ${drift.staleMappings.length}\n`);
return 0;
} finally {
await client.cleanup?.();
}
}
if (args.command === 'validate') {
const knownKloConnectionIds = new Set(Object.keys(project.config.connections));
const knownConnectionTypes = new Map(
Object.entries(project.config.connections).map(([id, _config]) => [id, targetPhysicalInfo(project, id).connection_type]),
);
const validation = validateLookerMappings({
mappings: await store.readMappings(args.connectionId),
knownKloConnectionIds,
knownConnectionTypes,
});
if (!validation.ok) {
for (const error of validation.errors) {
io.stderr.write(`${error.key}: ${error.reason}\n`);
}
return 1;
}
io.stdout.write(`Mapping validation passed: ${args.connectionId}\n`);
return 0;
}
if (args.command === 'clear') {
await store.clearConnectionMappings({
lookerConnectionId: args.connectionId,
lookerConnectionName: args.mappingKey ?? (args.metabaseDatabaseId ? String(args.metabaseDatabaseId) : undefined),
});
io.stdout.write(
args.mappingKey
? `Cleared connectionMappings.${args.mappingKey}\n`
: `Cleared mappings for ${args.connectionId}\n`,
);
return 0;
}
throw new Error(`Looker connection mapping does not support ${args.command}`);
}
assertMetabaseConnection(project, args.connectionId);
const store = new LocalMetabaseSourceStateReader({ dbPath: kloLocalStateDbPath(project) });
if (args.command === 'list') {
const rows = await store.listDatabaseMappings(args.connectionId);
io.stdout.write(args.json ? `${JSON.stringify(rows, null, 2)}\n` : `${rows.map(renderMapping).join('\n')}\n`);
return 0;
}
if (args.command === 'set') {
assertTargetConnection(project, args.value);
await store.upsertDatabaseMapping({
connectionId: args.connectionId,
metabaseDatabaseId: parseId(args.key, 'metabaseDatabaseId'),
targetConnectionId: args.value,
syncEnabled: true,
source: 'cli',
});
io.stdout.write(`Set databaseMappings.${args.key} = ${args.value}\n`);
return 0;
}
if (args.command === 'apply-bulk') {
const payload = JSON.parse(await readFile(args.filePath, 'utf8')) as MetabaseBulkMappingPayload;
const existingState = await store.getSourceState(args.connectionId);
const existingRows = await store.listDatabaseMappings(args.connectionId);
const existingById = new Map(existingRows.map((row) => [row.metabaseDatabaseId, row]));
const databaseMappings = payload.databaseMappings ?? {};
for (const targetConnectionId of Object.values(databaseMappings)) {
if (targetConnectionId) {
assertTargetConnection(project, targetConnectionId);
}
}
const mappingIds = new Set([
...existingRows.map((row) => row.metabaseDatabaseId),
...Object.keys(databaseMappings).map((id) => parseId(id, 'metabaseDatabaseId')),
...Object.keys(payload.syncEnabled ?? {}).map((id) => parseId(id, 'metabaseDatabaseId')),
]);
await store.replaceSourceState({
connectionId: args.connectionId,
syncMode: payload.syncMode ?? existingState.syncMode,
defaultTagNames: payload.defaultTagNames ?? existingState.defaultTagNames,
selections:
payload.selections === undefined
? existingState.selections
: [
...(payload.selections.collections ?? []).map((id) => ({
selectionType: 'collection' as const,
metabaseObjectId: id,
})),
...(payload.selections.items ?? []).map((id) => ({
selectionType: 'item' as const,
metabaseObjectId: id,
})),
],
mappings: [...mappingIds]
.sort((a, b) => a - b)
.map((id) => {
const existing = existingById.get(id);
return {
metabaseDatabaseId: id,
metabaseDatabaseName: existing?.metabaseDatabaseName ?? null,
metabaseEngine: existing?.metabaseEngine ?? null,
metabaseHost: existing?.metabaseHost ?? null,
metabaseDbName: existing?.metabaseDbName ?? null,
targetConnectionId: databaseMappings[String(id)] ?? existing?.targetConnectionId ?? null,
syncEnabled: payload.syncEnabled?.[String(id)] ?? existing?.syncEnabled ?? false,
source: 'cli',
};
}),
});
io.stdout.write(`Applied bulk mappings for ${args.connectionId}\n`);
return 0;
}
if (args.command === 'set-sync-enabled') {
await store.setMappingSyncEnabled({
connectionId: args.connectionId,
metabaseDatabaseId: args.metabaseDatabaseId,
syncEnabled: args.enabled,
});
io.stdout.write(`Set syncEnabled.${args.metabaseDatabaseId} = ${args.enabled}\n`);
return 0;
}
if (args.command === 'sync-state-get') {
const state = await store.getSourceState(args.connectionId);
const payload = {
syncMode: state.syncMode,
selections: state.selections,
defaultTagNames: state.defaultTagNames,
};
io.stdout.write(args.json ? `${JSON.stringify(payload, null, 2)}\n` : `${payload.syncMode}\n`);
return 0;
}
if (args.command === 'sync-state-set') {
await store.setSyncState({
connectionId: args.connectionId,
syncMode: args.syncMode,
defaultTagNames: args.tagNames,
selections: [
...args.collectionIds.map((id) => ({ selectionType: 'collection' as const, metabaseObjectId: id })),
...args.itemIds.map((id) => ({ selectionType: 'item' as const, metabaseObjectId: id })),
],
});
io.stdout.write(`Set sync state for ${args.connectionId}\n`);
return 0;
}
if (args.command === 'refresh') {
const client = await (deps.createMetabaseClient ?? createDefaultMetabaseClient)(project, args.connectionId);
try {
const discovered = await discoverMetabaseDatabases(client);
const existing = Object.fromEntries(
(await store.listDatabaseMappings(args.connectionId)).map((row) => [
String(row.metabaseDatabaseId),
row.targetConnectionId,
]),
);
const drift = computeMetabaseMappingDrift({ currentMappings: existing, discovered });
if (args.autoAccept) {
await store.refreshDiscoveredDatabases({ connectionId: args.connectionId, discovered });
}
io.stdout.write(`Discovery: ${discovered.length} ${discovered.length === 1 ? 'database' : 'databases'}\n`);
io.stdout.write(`Unmapped discovered: ${drift.unmappedDiscovered.length}\n`);
io.stdout.write(`Stale mappings: ${drift.staleMappings.length}\n`);
return 0;
} finally {
await client.cleanup();
}
}
if (args.command === 'validate') {
const rows = await store.listDatabaseMappings(args.connectionId);
const failures = rows.flatMap((row) => {
if (!row.targetConnectionId) {
return [];
}
const reason = validateMappingPhysicalMatch(
{ metabaseEngine: row.metabaseEngine, metabaseDbName: row.metabaseDbName, metabaseHost: row.metabaseHost },
project.config.connections[row.targetConnectionId]
? targetPhysicalInfo(project, row.targetConnectionId)
: { connection_type: 'UNKNOWN' },
);
return reason ? [`${row.metabaseDatabaseId}: ${reason}`] : [];
});
if (failures.length > 0) {
for (const failure of failures) {
io.stderr.write(`${failure}\n`);
}
return 1;
}
io.stdout.write(`Mapping validation passed: ${args.connectionId}\n`);
return 0;
}
const metabaseDatabaseId = args.metabaseDatabaseId ?? (args.mappingKey ? parseId(args.mappingKey, 'metabaseDatabaseId') : undefined);
await store.clearDatabaseMappings({ connectionId: args.connectionId, metabaseDatabaseId });
io.stdout.write(
metabaseDatabaseId
? `Cleared databaseMappings.${metabaseDatabaseId}\n`
: `Cleared mappings for ${args.connectionId}\n`,
);
return 0;
} catch (error) {
io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
return 1;
}
}

View file

@ -0,0 +1,132 @@
import { type Command, Option } from '@commander-js/extra-typings';
import {
type KloCliCommandContext,
parseNonEmptyAssignmentOption,
parsePositiveIntegerOption,
parseSafeConnectionIdOption,
resolveCommandProjectDir,
} from '../cli-program.js';
import {
type KloConnectionMetabaseSetupArgs,
type MetabaseSetupMappingAssignment,
type MetabaseSetupSyncMode,
runKloConnectionMetabaseSetup,
} from './connection-metabase-setup.js';
const SYNC_MODE_CHOICES = ['ALL', 'ONLY', 'EXCEPT'] as const satisfies readonly MetabaseSetupSyncMode[];
interface ConnectionMetabaseSetupOptions {
id?: string;
url?: string;
apiKey?: string;
mintApiKey?: boolean;
username?: string;
password?: string;
map: MetabaseSetupMappingAssignment[];
sync: number[];
syncMode: MetabaseSetupSyncMode;
runIngest?: boolean;
yes?: boolean;
input?: boolean;
}
function collectPositiveIntegerOption(value: string, previous: number[] = []): number[] {
return [...previous, parsePositiveIntegerOption(value)];
}
function parseMappingAssignment(value: string): MetabaseSetupMappingAssignment {
const assignment = parseNonEmptyAssignmentOption(value);
return {
metabaseDatabaseId: parsePositiveIntegerOption(assignment.key),
targetConnectionId: parseSafeConnectionIdOption(assignment.value),
};
}
function collectMappingOption(
value: string,
previous: MetabaseSetupMappingAssignment[] = [],
): MetabaseSetupMappingAssignment[] {
return [...previous, parseMappingAssignment(value)];
}
async function runMetabaseSetupArgs(
context: KloCliCommandContext,
args: KloConnectionMetabaseSetupArgs,
): Promise<void> {
const runner = context.deps.connectionMetabaseSetup ?? runKloConnectionMetabaseSetup;
context.setExitCode(await runner(args, context.io));
}
export function registerConnectionMetabaseCommands(connection: Command, context: KloCliCommandContext): void {
const metabase = connection
.command('metabase')
.description('Configure Metabase connections')
.showHelpAfterError()
.addHelpText(
'after',
'\nProject directory defaults to KLO_PROJECT_DIR when set, otherwise the current working directory.\n',
);
metabase.action(() => {
metabase.outputHelp();
context.setExitCode(0);
});
metabase
.command('setup')
.description('Guided setup for a Metabase connection')
.option('--id <connectionId>', 'KLO connection id to write', parseSafeConnectionIdOption)
.option('--url <url>', 'Metabase API URL')
.addOption(new Option('--api-key <key>', 'Metabase API key').conflicts('mintApiKey'))
.option('--mint-api-key', 'Mint a Metabase API key with credentials', false)
.option('--username <email>', 'Metabase admin username for API-key minting')
.option('--password <password>', 'Metabase admin password for API-key minting')
.addHelpText(
'after',
'\nGuided equivalent of:\n' +
' klo connection mapping refresh <connectionId> --auto-accept\n' +
' klo connection mapping set <connectionId> databaseMappings <id>=<target>\n' +
' klo connection mapping set-sync-enabled <connectionId> <id> --enabled true\n' +
' klo ingest <connectionId>\n',
)
.option(
'--map <metabaseDatabaseId=targetConnectionId>',
'Assign a Metabase database id to a warehouse connection; repeatable',
collectMappingOption,
[],
)
.option(
'--sync <metabaseDatabaseId>',
'Enable Metabase sync for a discovered database; repeatable',
collectPositiveIntegerOption,
[],
)
.addOption(
new Option('--sync-mode <mode>', 'Metabase sync selection mode')
.choices(SYNC_MODE_CHOICES)
.default('ALL' satisfies MetabaseSetupSyncMode),
)
.option('--run-ingest', 'Run ingest after setup', false)
.option('--yes', 'Confirm and apply setup changes without prompting', false)
.option('--no-input', 'Disable interactive terminal input')
.showHelpAfterError()
.action(async (options: ConnectionMetabaseSetupOptions, command) => {
await runMetabaseSetupArgs(context, {
command: 'setup',
projectDir: resolveCommandProjectDir(command),
connectionId: options.id,
url: options.url,
apiKey: options.apiKey,
mintApiKey: options.mintApiKey === true,
metabaseUsername: options.username,
metabasePassword: options.password,
mappings: options.map,
syncEnabledDatabaseIds: options.sync,
syncMode: options.syncMode ?? 'ALL',
runIngest: options.runIngest === true,
yes: options.yes === true,
inputMode: options.input === false ? 'disabled' : 'auto',
});
});
}

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,782 @@
import type { Option as ClackOption } from '@clack/prompts';
import {
cancel,
confirm,
intro,
isCancel,
log,
multiselect,
note,
outro,
password,
select,
text,
} from '@clack/prompts';
import { localConnectionToWarehouseDescriptor } from '@klo/context/connections';
import {
DEFAULT_METABASE_CLIENT_CONFIG,
DefaultMetabaseConnectionClientFactory,
LocalMetabaseSourceStateReader,
MetabaseClient,
type MetabaseDatabase,
type MetabaseRuntimeClient,
type MetabaseSyncMode,
metabaseRuntimeConfigFromLocalConnection,
validateMappingPhysicalMatch,
} from '@klo/context/ingest';
import {
type KloLocalProject,
type KloProjectConnectionConfig,
kloLocalStateDbPath,
loadKloProject,
serializeKloProjectConfig,
} from '@klo/context/project';
import { createClackSpinner, type KloCliSpinner } from '../clack.js';
import type { KloCliIo } from '../cli-runtime.js';
import { withMenuOptionsSpacing, withMultiselectNavigation } from '../prompt-navigation.js';
import { type KloPublicIngestArgs, runKloPublicIngest } from '../public-ingest.js';
export type KloMetabaseSetupInputMode = 'auto' | 'disabled';
export type MetabaseSetupSyncMode = MetabaseSyncMode;
type MetabaseSetupPromptOption<Value> = ClackOption<Value>;
export interface MetabaseSetupLogger {
info(message: string): void;
step(message: string): void;
success(message: string): void;
warn(message: string): void;
error(message: string): void;
}
export interface MetabaseSetupPromptAdapter {
intro(title?: string): void;
outro(message?: string): void;
note(message: string, title: string): void;
log: MetabaseSetupLogger;
spinner(): KloCliSpinner;
select<T extends string>(options: { message: string; options: Array<MetabaseSetupPromptOption<T>> }): Promise<T>;
multiselect<Value extends number | string>(options: {
message: string;
options: Array<MetabaseSetupPromptOption<Value>>;
initialValues?: Value[];
required?: boolean;
maxItems?: number;
}): Promise<Value[]>;
text(options: { message: string; placeholder?: string }): Promise<string>;
password(options: { message: string }): Promise<string>;
confirm(options: { message: string; initialValue?: boolean }): Promise<boolean>;
cancel(message: string): void;
}
type KloMetabaseSetupInteractiveIo = KloCliIo & {
stdin?: { isTTY?: boolean };
};
export interface MetabaseSetupMappingAssignment {
metabaseDatabaseId: number;
targetConnectionId: string;
}
export interface MintMetabaseApiKeyArgs {
url: string;
username: string;
password: string;
}
export type MintMetabaseApiKey = (args: MintMetabaseApiKeyArgs, io: KloCliIo) => Promise<string>;
export interface KloConnectionMetabaseSetupArgs {
command: 'setup';
projectDir: string;
connectionId?: string;
url?: string;
apiKey?: string;
mintApiKey: boolean;
metabaseUsername?: string;
metabasePassword?: string;
mappings: MetabaseSetupMappingAssignment[];
syncEnabledDatabaseIds: number[];
syncMode: MetabaseSetupSyncMode;
runIngest: boolean;
yes: boolean;
inputMode: KloMetabaseSetupInputMode;
}
export interface KloConnectionMetabaseSetupDeps {
createMetabaseClient?: (
project: KloLocalProject,
connectionId: string,
) => Promise<Pick<MetabaseRuntimeClient, 'testConnection' | 'getDatabases' | 'cleanup'>>;
mintMetabaseApiKey?: MintMetabaseApiKey;
prompts?: MetabaseSetupPromptAdapter;
runPublicIngest?: (args: Extract<KloPublicIngestArgs, { command: 'run' }>, io: KloCliIo) => Promise<number>;
}
function isMetabaseConnection(connection: KloProjectConnectionConfig | undefined): boolean {
return (
String(connection?.driver ?? '')
.trim()
.toLowerCase() === 'metabase'
);
}
function stringField(value: unknown): string | undefined {
return typeof value === 'string' && value.trim().length > 0 ? value.trim() : undefined;
}
function uniqueSorted(values: number[]): number[] {
return [...new Set(values)].sort((a, b) => a - b);
}
function resolveMetabaseUrl(connection: KloProjectConnectionConfig | undefined): string | undefined {
return stringField(connection?.api_url) ?? stringField(connection?.apiUrl) ?? stringField(connection?.url);
}
function resolveLiteralMetabaseApiKey(connection: KloProjectConnectionConfig | undefined): string | undefined {
return stringField(connection?.api_key) ?? stringField(connection?.apiKey);
}
function listMetabaseConnectionIds(project: KloLocalProject): string[] {
return Object.entries(project.config.connections)
.filter(([_connectionId, connection]) => isMetabaseConnection(connection))
.map(([connectionId]) => connectionId)
.sort();
}
function listWarehouseConnectionIds(project: KloLocalProject): string[] {
return Object.entries(project.config.connections)
.filter(([connectionId, connection]) => localConnectionToWarehouseDescriptor(connectionId, connection) != null)
.map(([connectionId]) => connectionId)
.sort();
}
function redactSecrets(message: string, secrets: string[]): string {
let result = message;
for (const secret of secrets) {
if (!secret) {
continue;
}
result = result.split(secret).join('[redacted]');
}
return result;
}
async function createDefaultMetabaseClient(
project: KloLocalProject,
connectionId: string,
): Promise<Pick<MetabaseRuntimeClient, 'testConnection' | 'getDatabases' | 'cleanup'>> {
const factory = new DefaultMetabaseConnectionClientFactory(
(metabaseConnectionId) =>
metabaseRuntimeConfigFromLocalConnection(metabaseConnectionId, project.config.connections[metabaseConnectionId]),
DEFAULT_METABASE_CLIENT_CONFIG,
);
return factory.createClient(connectionId);
}
async function defaultMintMetabaseApiKey(args: MintMetabaseApiKeyArgs): Promise<string> {
const loginClient = new MetabaseClient({ apiUrl: args.url, apiKey: '' }, DEFAULT_METABASE_CLIENT_CONFIG);
const sessionId = await loginClient.createSession(args.username, args.password);
const sessionClient = new MetabaseClient(
{ apiUrl: args.url, apiKey: sessionId, authHeaderName: 'X-Metabase-Session' },
DEFAULT_METABASE_CLIENT_CONFIG,
);
const groups = await sessionClient.getPermissionGroups();
const adminGroup = groups.find((group) => group.name === 'Administrators');
if (!adminGroup) {
throw new Error('Metabase Administrators group was not found; create an API key manually and pass --api-key');
}
const mintedKey = await sessionClient.createApiKey({
groupId: adminGroup.id,
name: `KLO CLI ${new Date().toISOString()}`,
});
const trimmedKey = stringField(mintedKey);
if (!trimmedKey) {
throw new Error('Metabase API key minting returned an empty key');
}
return trimmedKey;
}
function ensureNotCancelled<T>(value: T | symbol, prompts: Pick<MetabaseSetupPromptAdapter, 'cancel'>): T {
if (isCancel(value)) {
prompts.cancel('Setup cancelled.');
throw new Error('Setup cancelled.');
}
return value as T;
}
export function createClackMetabaseSetupPromptAdapter(): MetabaseSetupPromptAdapter {
return {
intro(title?: string): void {
intro(title);
},
outro(message?: string): void {
outro(message);
},
note(message: string, title: string): void {
note(message, title);
},
log: {
info(message: string): void {
log.info(message);
},
step(message: string): void {
log.step(message);
},
success(message: string): void {
log.success(message);
},
warn(message: string): void {
log.warn(message);
},
error(message: string): void {
log.error(message);
},
},
spinner(): KloCliSpinner {
return createClackSpinner();
},
async select<T extends string>(options: {
message: string;
options: Array<MetabaseSetupPromptOption<T>>;
}): Promise<T> {
return ensureNotCancelled(await select(withMenuOptionsSpacing(options)), this);
},
async multiselect<Value extends number | string>(options: {
message: string;
options: Array<MetabaseSetupPromptOption<Value>>;
initialValues?: Value[];
required?: boolean;
maxItems?: number;
}): Promise<Value[]> {
return ensureNotCancelled(await multiselect(withMenuOptionsSpacing(options)), this);
},
async text(options: { message: string; placeholder?: string }): Promise<string> {
return ensureNotCancelled(await text(options), this);
},
async password(options: { message: string }): Promise<string> {
return ensureNotCancelled(await password(options), this);
},
async confirm(options: { message: string; initialValue?: boolean }): Promise<boolean> {
return ensureNotCancelled(await confirm(options), this);
},
cancel(message: string): void {
cancel(message);
},
};
}
function isInteractiveMetabaseSetupIo(
args: Pick<KloConnectionMetabaseSetupArgs, 'inputMode'>,
io: KloMetabaseSetupInteractiveIo,
): boolean {
return args.inputMode !== 'disabled' && io.stdin?.isTTY === true && io.stdout.isTTY === true;
}
function normalizeDiscoveredDatabases(databases: MetabaseDatabase[]): Array<{
id: number;
name: string;
engine: string;
host: string | null;
dbName: string | null;
}> {
return databases
.filter((database) => database.is_sample !== true)
.map((database) => ({
id: database.id,
name: database.name,
engine: stringField(database.engine) ?? 'unknown',
host: stringField(database.details?.host) ?? null,
dbName: stringField(database.details?.dbname) ?? null,
}));
}
function targetPhysicalInfo(project: KloLocalProject, connectionId: string) {
const descriptor = localConnectionToWarehouseDescriptor(connectionId, project.config.connections[connectionId]);
if (!descriptor) {
return { connection_type: 'UNKNOWN' };
}
return {
connection_type: descriptor.connection_type,
host: descriptor.host ?? null,
database: descriptor.database ?? null,
account: descriptor.account ?? null,
project_id: descriptor.project_id ?? null,
dataset_id: descriptor.dataset_id ?? null,
...descriptor.connection_params,
};
}
function noteMetabaseSetupSummary(options: {
prompts: MetabaseSetupPromptAdapter;
connectionId: string;
url: string;
mappings: MetabaseSetupMappingAssignment[];
syncEnabledDatabaseIds: number[];
}): void {
const mappingLines = options.mappings
.map((mapping) => ` ${mapping.metabaseDatabaseId} -> ${mapping.targetConnectionId}`)
.join('\n');
const syncLines = options.syncEnabledDatabaseIds.map((id) => ` ${id}`).join('\n');
options.prompts.note(
[
`Connection: ${options.connectionId}`,
`URL: ${options.url}`,
'',
'Mappings:',
mappingLines || ' (none)',
'',
'Sync enabled:',
syncLines || ' (none)',
].join('\n'),
'Summary',
);
}
export async function runKloConnectionMetabaseSetup(
args: KloConnectionMetabaseSetupArgs,
io: KloCliIo,
deps: KloConnectionMetabaseSetupDeps = {},
): Promise<number> {
let apiKeyForRedaction = args.apiKey;
let passwordForRedaction = args.metabasePassword;
const interactiveIo = io as KloMetabaseSetupInteractiveIo;
const isInteractive = isInteractiveMetabaseSetupIo(args, interactiveIo);
const prompts = deps.prompts ?? (isInteractive ? createClackMetabaseSetupPromptAdapter() : undefined);
try {
if (isInteractive && prompts) {
prompts.intro('KLO Metabase setup');
}
const project = await loadKloProject({ projectDir: args.projectDir });
const existingMetabaseConnectionIds = listMetabaseConnectionIds(project);
let connectionId: string;
if (args.connectionId) {
connectionId = args.connectionId;
} else if (existingMetabaseConnectionIds.length === 1) {
const onlyMetabaseConnectionId = existingMetabaseConnectionIds[0];
if (!onlyMetabaseConnectionId) {
throw new Error('No Metabase connection id was resolved');
}
connectionId = onlyMetabaseConnectionId;
} else if (existingMetabaseConnectionIds.length > 1) {
if (!isInteractive || !prompts) {
throw new Error(
`Multiple Metabase connections found (${existingMetabaseConnectionIds.join(', ')}); select one with --id`,
);
}
connectionId = await prompts.select({
message: 'Select the Metabase connection to configure',
options: existingMetabaseConnectionIds.map((id) => ({ value: id, label: id })),
});
} else {
connectionId = 'metabase';
}
const existingConnection = project.config.connections[connectionId];
const warehouseConnectionIds = listWarehouseConnectionIds(project);
if (warehouseConnectionIds.length === 0) {
throw new Error('Add a warehouse connection first');
}
let url = args.url ?? resolveMetabaseUrl(existingConnection);
let apiKey = args.apiKey ?? resolveLiteralMetabaseApiKey(existingConnection);
apiKeyForRedaction = apiKey;
if (!url && isInteractive && prompts) {
url = stringField(
await prompts.text({
message: 'Metabase API URL',
placeholder: 'http://localhost:3000',
}),
);
}
if (args.inputMode === 'disabled' && !url) {
throw new Error('missing Metabase URL');
}
if (!args.apiKey && !args.mintApiKey && apiKey && isInteractive && prompts && !args.yes) {
const reuse = await prompts.confirm({
message: `Reuse the existing Metabase API key from connections.${connectionId}?`,
initialValue: true,
});
if (!reuse) {
apiKey = undefined;
apiKeyForRedaction = undefined;
}
}
if (args.mintApiKey) {
let username = stringField(args.metabaseUsername);
let metabasePassword = stringField(args.metabasePassword);
if (isInteractive && prompts) {
if (!username) {
username = stringField(await prompts.text({ message: 'Metabase admin username' }));
}
if (!metabasePassword) {
metabasePassword = stringField(await prompts.password({ message: 'Metabase admin password' }));
}
}
if (!username) {
throw new Error('--mint-api-key requires --username');
}
if (!metabasePassword) {
throw new Error('--mint-api-key requires --password');
}
if (!url) {
throw new Error('Metabase URL is required (use --url)');
}
passwordForRedaction = metabasePassword;
apiKey = await (deps.mintMetabaseApiKey ?? defaultMintMetabaseApiKey)(
{ url, username, password: metabasePassword },
io,
);
apiKeyForRedaction = apiKey;
}
if (!apiKey && isInteractive && prompts) {
const credentialMode = await prompts.select({
message: 'Metabase credentials',
options: [
{ value: 'paste', label: 'Paste API key' },
{ value: 'mint', label: 'Mint API key' },
],
});
if (credentialMode === 'paste') {
apiKey = stringField(await prompts.password({ message: 'Metabase API key' }));
apiKeyForRedaction = apiKey;
} else {
const username = stringField(await prompts.text({ message: 'Metabase admin username' }));
const metabasePassword = stringField(await prompts.password({ message: 'Metabase admin password' }));
if (!username) {
throw new Error('Metabase username is required');
}
if (!metabasePassword) {
throw new Error('Metabase password is required');
}
if (!url) {
throw new Error('Metabase URL is required (use --url)');
}
passwordForRedaction = metabasePassword;
apiKey = await (deps.mintMetabaseApiKey ?? defaultMintMetabaseApiKey)(
{ url, username, password: metabasePassword },
io,
);
apiKeyForRedaction = apiKey;
}
}
if (args.inputMode === 'disabled' && !apiKey) {
throw new Error('missing Metabase API key');
}
if (!url) {
throw new Error('Metabase URL is required (use --url)');
}
if (!apiKey) {
throw new Error('Metabase API key is required (use --api-key)');
}
const transientConnectionConfig: KloProjectConnectionConfig = {
...(existingConnection ?? {}),
driver: 'metabase',
api_url: url,
api_key: apiKey,
};
const configWithTransient = {
...project.config,
connections: {
...project.config.connections,
[connectionId]: transientConnectionConfig,
},
};
const discoveryProject: KloLocalProject = { ...project, config: configWithTransient };
for (const mapping of args.mappings) {
if (!configWithTransient.connections[mapping.targetConnectionId]) {
throw new Error(`Target connection "${mapping.targetConnectionId}" does not exist`);
}
}
const client = await (deps.createMetabaseClient ?? createDefaultMetabaseClient)(discoveryProject, connectionId);
try {
const authSpinner = isInteractive && prompts ? prompts.spinner() : undefined;
authSpinner?.start('Testing Metabase connection');
const testResult = await client.testConnection();
if (!testResult.success) {
authSpinner?.error('Metabase authentication failed');
throw new Error(
`Metabase authentication failed. Replace connections.${connectionId}.api_key or use --mint-api-key.`,
);
}
authSpinner?.stop('Metabase reachable');
const discoverySpinner = isInteractive && prompts ? prompts.spinner() : undefined;
discoverySpinner?.start('Discovering Metabase databases');
const discovered = normalizeDiscoveredDatabases(await client.getDatabases());
discoverySpinner?.stop(`Discovered ${discovered.length} ${discovered.length === 1 ? 'database' : 'databases'}`);
if (isInteractive && prompts) {
prompts.log.success(
`Discovered ${discovered.length} ${discovered.length === 1 ? 'database' : 'databases'}`,
);
}
if (discovered.length === 0) {
throw new Error('Metabase auth worked but no usable databases were returned');
}
let resolvedMappings = args.mappings;
let resolvedSyncEnabledDatabaseIds = args.syncEnabledDatabaseIds;
if (resolvedSyncEnabledDatabaseIds.length === 0 && args.yes && resolvedMappings.length > 0) {
resolvedSyncEnabledDatabaseIds = uniqueSorted(resolvedMappings.map((mapping) => mapping.metabaseDatabaseId));
}
if (resolvedMappings.length === 0 && resolvedSyncEnabledDatabaseIds.length === 0) {
const onlyDiscoveredDatabase = discovered.length === 1 ? discovered[0] : undefined;
const compatibleWarehouses = onlyDiscoveredDatabase
? warehouseConnectionIds.filter((warehouseConnectionId) => {
const mismatchReason = validateMappingPhysicalMatch(
{
metabaseEngine: onlyDiscoveredDatabase.engine,
metabaseDbName: onlyDiscoveredDatabase.dbName,
metabaseHost: onlyDiscoveredDatabase.host,
},
targetPhysicalInfo(project, warehouseConnectionId),
);
return !mismatchReason;
})
: [];
const onlyWarehouseConnectionId = compatibleWarehouses[0];
if (onlyDiscoveredDatabase && compatibleWarehouses.length === 1 && onlyWarehouseConnectionId) {
if (args.yes) {
resolvedMappings = [
{ metabaseDatabaseId: onlyDiscoveredDatabase.id, targetConnectionId: onlyWarehouseConnectionId },
];
resolvedSyncEnabledDatabaseIds = [onlyDiscoveredDatabase.id];
} else if (isInteractive && prompts) {
const proposedMappings = [
{ metabaseDatabaseId: onlyDiscoveredDatabase.id, targetConnectionId: onlyWarehouseConnectionId },
];
const proposedSyncEnabledDatabaseIds = [onlyDiscoveredDatabase.id];
noteMetabaseSetupSummary({
prompts,
connectionId,
url,
mappings: proposedMappings,
syncEnabledDatabaseIds: proposedSyncEnabledDatabaseIds,
});
const confirmed = await prompts.confirm({
message: `Map Metabase database "${onlyDiscoveredDatabase.name}" (${onlyDiscoveredDatabase.id}) to "${onlyWarehouseConnectionId}" and enable sync?`,
initialValue: true,
});
if (!confirmed) {
prompts.cancel('Setup cancelled.');
throw new Error('Setup cancelled.');
}
resolvedMappings = proposedMappings;
resolvedSyncEnabledDatabaseIds = proposedSyncEnabledDatabaseIds;
} else {
throw new Error('Metabase mapping/sync is required in --no-input mode; pass --map and --sync');
}
} else if (isInteractive && prompts) {
const selectedDatabaseIds = await prompts.multiselect<number>({
message: withMultiselectNavigation('Select Metabase databases to configure'),
options: discovered.map((database) => ({
value: database.id,
label: `${database.id}: ${database.name}`,
hint: [database.engine, database.host, database.dbName].filter(Boolean).join(' • '),
})),
required: true,
});
resolvedMappings = [];
for (const databaseId of selectedDatabaseIds) {
const database = discovered.find((candidate) => candidate.id === databaseId);
if (!database) {
throw new Error(`Selected database id ${databaseId} was not discovered`);
}
const existingMapping = args.mappings.find((mapping) => mapping.metabaseDatabaseId === databaseId);
if (existingMapping) {
resolvedMappings.push(existingMapping);
continue;
}
const targetConnectionId = await prompts.select({
message: `Map Metabase database ${database.id} ("${database.name}") to which KLO connection?`,
options: warehouseConnectionIds.map((warehouseId) => ({ value: warehouseId, label: warehouseId })),
});
resolvedMappings.push({ metabaseDatabaseId: databaseId, targetConnectionId });
}
const syncIds = await prompts.multiselect<number>({
message: withMultiselectNavigation('Enable sync for which databases?'),
options: selectedDatabaseIds.map((id) => ({ value: id, label: String(id) })),
initialValues: selectedDatabaseIds,
required: true,
});
resolvedSyncEnabledDatabaseIds = uniqueSorted(syncIds);
if (!args.yes) {
noteMetabaseSetupSummary({
prompts,
connectionId,
url,
mappings: resolvedMappings,
syncEnabledDatabaseIds: resolvedSyncEnabledDatabaseIds,
});
const confirmed = await prompts.confirm({
message: 'Write changes to klo.yaml and enable sync?',
initialValue: true,
});
if (!confirmed) {
prompts.cancel('Setup cancelled.');
throw new Error('Setup cancelled.');
}
}
} else if (args.inputMode === 'disabled') {
throw new Error('Metabase mapping/sync is required in --no-input mode; pass --map and --sync');
}
}
if (
args.inputMode === 'disabled' &&
resolvedMappings.length > 0 &&
resolvedSyncEnabledDatabaseIds.length === 0
) {
throw new Error('Metabase sync selection is required in --no-input mode; pass --sync <metabaseDatabaseId>');
}
const discoveredIds = new Set(discovered.map((database) => database.id));
for (const mapping of resolvedMappings) {
if (!discoveredIds.has(mapping.metabaseDatabaseId)) {
throw new Error(`Mapped database id ${mapping.metabaseDatabaseId} was not discovered`);
}
}
for (const syncId of resolvedSyncEnabledDatabaseIds) {
if (!discoveredIds.has(syncId)) {
throw new Error(`Sync database id ${syncId} was not discovered`);
}
}
await project.fileStore.writeFile(
'klo.yaml',
serializeKloProjectConfig(configWithTransient),
'klo',
'klo@example.com',
`Setup Metabase connection ${connectionId}`,
);
const updatedProject = await loadKloProject({ projectDir: args.projectDir });
const store = new LocalMetabaseSourceStateReader({ dbPath: kloLocalStateDbPath(updatedProject) });
await store.refreshDiscoveredDatabases({ connectionId, discovered });
for (const mapping of resolvedMappings) {
await store.upsertDatabaseMapping({
connectionId,
metabaseDatabaseId: mapping.metabaseDatabaseId,
targetConnectionId: mapping.targetConnectionId,
syncEnabled: false,
source: 'cli',
});
}
for (const metabaseDatabaseId of resolvedSyncEnabledDatabaseIds) {
await store.setMappingSyncEnabled({
connectionId,
metabaseDatabaseId,
syncEnabled: true,
});
}
const existingSyncState = await store.getSourceState(connectionId);
await store.setSyncState({
connectionId,
syncMode: args.syncMode,
defaultTagNames: existingSyncState.defaultTagNames,
selections: existingSyncState.selections,
});
const unhydrated = await store.getUnhydratedSyncEnabledMappingIds(connectionId);
if (unhydrated.length > 0) {
io.stderr.write(
`Sync-enabled mappings are missing discovery metadata; run klo connection mapping refresh ${connectionId} --auto-accept\n`,
);
return 1;
}
const rows = await store.listDatabaseMappings(connectionId);
const physicalFailures = rows.flatMap((row) => {
if (!row.targetConnectionId) {
return [];
}
const reason = validateMappingPhysicalMatch(
{ metabaseEngine: row.metabaseEngine, metabaseDbName: row.metabaseDbName, metabaseHost: row.metabaseHost },
updatedProject.config.connections[row.targetConnectionId]
? targetPhysicalInfo(updatedProject, row.targetConnectionId)
: { connection_type: 'UNKNOWN' },
);
return reason ? [`${row.metabaseDatabaseId}: ${reason}`] : [];
});
if (physicalFailures.length > 0) {
for (const failure of physicalFailures) {
io.stderr.write(`${failure}\n`);
}
return 1;
}
io.stdout.write(`Connection: ${connectionId}\n`);
io.stdout.write(`Discovered ${discovered.length} ${discovered.length === 1 ? 'database' : 'databases'}\n`);
io.stdout.write(`Next: klo ingest ${connectionId} --project-dir ${args.projectDir}\n`);
if (args.runIngest) {
const ingestRunner = deps.runPublicIngest ?? runKloPublicIngest;
const exitCode = await ingestRunner(
{
command: 'run',
projectDir: args.projectDir,
targetConnectionId: connectionId,
all: false,
json: false,
inputMode: 'disabled',
},
io,
);
if (exitCode !== 0) {
io.stderr.write(`Ingest failed; re-run: klo ingest ${connectionId} --project-dir ${args.projectDir}\n`);
return 1;
}
}
if (isInteractive && prompts) {
prompts.outro('Metabase setup complete');
}
return 0;
} finally {
await client.cleanup();
}
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
io.stderr.write(
`${redactSecrets(message, [apiKeyForRedaction ?? '', passwordForRedaction ?? '', args.apiKey ?? ''])}\n`,
);
return 1;
}
}

View file

@ -0,0 +1,92 @@
import { type Command, InvalidArgumentError } from '@commander-js/extra-typings';
import { collectOption, type KloCliCommandContext, resolveCommandProjectDir } from '../cli-program.js';
import type { KloConnectionNotionArgs } from './connection-notion.js';
interface NotionPickOptions {
input?: boolean;
rootPageId: string[];
}
function parseSafeConnectionId(value: string): string {
if (!/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(value)) {
throw new InvalidArgumentError(`Unsafe connection id: ${value}`);
}
return value;
}
function uniqueInOrder(values: string[]): string[] {
const seen = new Set<string>();
const result: string[] = [];
for (const value of values) {
if (!seen.has(value)) {
seen.add(value);
result.push(value);
}
}
return result;
}
function normalizeNotionPageId(value: string): string {
const trimmed = value.trim();
const compact = trimmed.includes('-') ? trimmed.replace(/-/g, '') : trimmed;
if (!/^[0-9a-fA-F]{32}$/.test(compact)) {
throw new Error(`Invalid Notion page UUID: ${value}`);
}
const lower = compact.toLowerCase();
return `${lower.slice(0, 8)}-${lower.slice(8, 12)}-${lower.slice(12, 16)}-${lower.slice(16, 20)}-${lower.slice(20)}`;
}
function buildPickArgs(connectionId: string, projectDir: string, options: NotionPickOptions): KloConnectionNotionArgs {
if (options.input !== false) {
return {
command: 'pick',
projectDir,
connectionId,
mode: 'interactive',
};
}
const rootPageIds = uniqueInOrder(options.rootPageId.map(normalizeNotionPageId));
if (rootPageIds.length === 0) {
throw new Error('connection notion pick --no-input requires at least one --root-page-id');
}
return {
command: 'pick',
projectDir,
connectionId,
mode: 'non-interactive',
rootPageIds,
};
}
async function runConnectionNotionArgs(context: KloCliCommandContext, args: KloConnectionNotionArgs): Promise<void> {
const runner = context.deps.connectionNotion ?? (await import('./connection-notion.js')).runKloConnectionNotion;
context.setExitCode(await runner(args, context.io));
}
export function registerConnectionNotionCommands(connect: Command, context: KloCliCommandContext): void {
const notion = connect
.command('notion')
.description('Configure Notion source selection')
.showHelpAfterError()
.addHelpText(
'after',
'\nProject directory defaults to KLO_PROJECT_DIR when set, otherwise the current working directory.\n',
);
notion.action(() => {
notion.outputHelp();
context.setExitCode(0);
});
notion
.command('pick')
.description('Pick Notion root pages for a configured Notion connection')
.argument('<connectionId>', 'Notion connection id', parseSafeConnectionId)
.option('--no-input', 'Disable interactive terminal input')
.option('--root-page-id <id>', 'Root page UUID to crawl; repeatable with --no-input', collectOption, [])
.showHelpAfterError()
.action(async (connectionId: string, options: NotionPickOptions, command) => {
await runConnectionNotionArgs(context, buildPickArgs(connectionId, resolveCommandProjectDir(command), options));
});
}

View file

@ -0,0 +1,283 @@
import { describe, expect, it } from 'vitest';
import {
buildInitialState,
buildPickerTree,
canToggle,
clearExpiredTransientHint,
filterTree,
flattenSelection,
moveCursor,
reducer,
selectAllVisible,
selectNone,
toggleChecked,
TRANSIENT_HINT_DURATION_MS,
visibleNodeIds,
type NotionPickerPageInput,
} from './connection-notion-tree.js';
const IDS = {
engineering: '11111111-1111-1111-1111-111111111111',
architecture: '22222222-2222-2222-2222-222222222222',
onboarding: '33333333-3333-3333-3333-333333333333',
marketing: '44444444-4444-4444-4444-444444444444',
journal: '55555555-5555-5555-5555-555555555555',
orphan: '66666666-6666-6666-6666-666666666666',
duplicate: '77777777-7777-7777-7777-777777777777',
cycleA: '88888888-8888-8888-8888-888888888888',
cycleB: '99999999-9999-9999-9999-999999999999',
};
function pages(): NotionPickerPageInput[] {
return [
{ id: IDS.marketing, title: 'Marketing', archived: false, parentId: null },
{ id: IDS.onboarding, title: 'Onboarding', archived: false, parentId: IDS.engineering },
{ id: IDS.engineering, title: 'Engineering Docs', archived: false, parentId: null },
{ id: IDS.architecture, title: 'Architecture', archived: false, parentId: IDS.engineering },
{ id: IDS.journal, title: 'Daily journal', archived: true, parentId: IDS.marketing },
{ id: IDS.orphan, title: '', archived: false, parentId: 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa' },
{ id: IDS.duplicate, title: 'Original duplicate', archived: false, parentId: null },
{ id: IDS.duplicate, title: 'Ignored duplicate', archived: true, parentId: IDS.marketing },
{ id: IDS.cycleA, title: 'Cycle A', archived: false, parentId: IDS.cycleB },
{ id: IDS.cycleB, title: 'Cycle B', archived: false, parentId: IDS.cycleA },
];
}
describe('buildPickerTree', () => {
it('deduplicates pages, sorts siblings, preserves archived flags, roots orphans, and breaks cycles', () => {
const tree = buildPickerTree(pages());
const byId = new Map(tree.map((node) => [node.id, node]));
expect(tree.map((node) => node.title)).toEqual([
'Cycle A',
'Cycle B',
'Engineering Docs',
'Architecture',
'Onboarding',
'Marketing',
'Daily journal',
'Original duplicate',
'Untitled',
]);
expect(byId.get(IDS.engineering)?.childIds).toEqual([IDS.architecture, IDS.onboarding]);
expect(byId.get(IDS.architecture)).toMatchObject({
depth: 1,
parentId: IDS.engineering,
path: 'Engineering Docs / Architecture',
});
expect(byId.get(IDS.journal)).toMatchObject({
archived: true,
depth: 1,
path: 'Marketing / Daily journal',
});
expect(byId.get(IDS.orphan)).toMatchObject({
title: 'Untitled',
parentId: null,
depth: 0,
path: 'Untitled',
});
expect(byId.get(IDS.duplicate)).toMatchObject({
title: 'Original duplicate',
archived: false,
parentId: null,
});
expect(byId.get(IDS.cycleA)?.parentId).toBeNull();
expect(byId.get(IDS.cycleB)?.parentId).toBe(IDS.cycleA);
});
});
describe('selection invariants', () => {
it('checking a parent locks descendants and keeps checked ids minimal', () => {
const state = buildInitialState({
tree: buildPickerTree(pages()),
existingRootPageIds: [],
currentCrawlMode: 'selected_roots',
});
const checkedParent = toggleChecked(state, IDS.engineering, 1000);
expect([...checkedParent.checked]).toEqual([IDS.engineering]);
expect(canToggle(IDS.architecture, checkedParent)).toEqual({
ok: false,
reason: "Locked by 'Engineering Docs' - uncheck parent first",
});
const lockedChildAttempt = toggleChecked(checkedParent, IDS.architecture, 2000);
expect([...lockedChildAttempt.checked]).toEqual([IDS.engineering]);
expect(lockedChildAttempt.transientHint).toEqual({
text: "Locked by 'Engineering Docs' - uncheck parent first",
expiresAt: 4500,
});
const uncheckedParent = toggleChecked(lockedChildAttempt, IDS.engineering, 3000);
expect([...uncheckedParent.checked]).toEqual([]);
expect(canToggle(IDS.architecture, uncheckedParent)).toEqual({ ok: true });
});
it('normalizes stored roots, reports stale roots, expands checked ancestors, and flattens descendants', () => {
const state = buildInitialState({
tree: buildPickerTree(pages()),
existingRootPageIds: [
IDS.engineering.replaceAll('-', ''),
IDS.architecture,
'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa',
],
currentCrawlMode: 'selected_roots',
});
expect([...state.checked]).toEqual([IDS.engineering]);
expect([...state.expanded]).toEqual([]);
expect(state.cursorId).toBe(IDS.cycleA);
expect(state.preLoadWarnings).toEqual(['1 stored root_page_ids no longer visible']);
expect(flattenSelection(new Set([IDS.engineering, IDS.architecture]), state.byId)).toEqual([IDS.engineering]);
});
});
describe('search and cursor movement', () => {
it('filters by title and path while deriving auto-expanded ancestors', () => {
const state = buildInitialState({
tree: buildPickerTree(pages()),
existingRootPageIds: [],
currentCrawlMode: 'selected_roots',
});
const searching = {
...state,
search: { editing: false, query: 'architecture' },
};
expect(filterTree(searching)).toEqual({
visibleIds: new Set([IDS.engineering, IDS.architecture]),
autoExpand: new Set([IDS.engineering]),
});
expect(visibleNodeIds(searching)).toEqual([IDS.engineering, IDS.architecture]);
});
it('moves the cursor through visible nodes and implements left/right tree semantics', () => {
const state = buildInitialState({
tree: buildPickerTree(pages()),
existingRootPageIds: [],
currentCrawlMode: 'selected_roots',
});
const atEngineering = {
...state,
cursorId: IDS.engineering,
expanded: new Set([IDS.engineering]),
};
expect(moveCursor(atEngineering, 'down').cursorId).toBe(IDS.architecture);
expect(moveCursor({ ...atEngineering, cursorId: IDS.architecture }, 'up').cursorId).toBe(IDS.engineering);
expect(moveCursor(atEngineering, 'right').cursorId).toBe(IDS.architecture);
expect(moveCursor({ ...atEngineering, cursorId: IDS.architecture }, 'left').cursorId).toBe(IDS.engineering);
expect([...moveCursor(atEngineering, 'left').expanded]).toEqual([]);
expect([...moveCursor({ ...state, cursorId: IDS.marketing }, 'right').expanded]).toContain(IDS.marketing);
});
});
describe('bulk actions and reducer effects', () => {
it('selects only matching visible roots under search and clears selection', () => {
const state = buildInitialState({
tree: buildPickerTree(pages()),
existingRootPageIds: [IDS.marketing],
currentCrawlMode: 'selected_roots',
});
const searching = {
...state,
search: { editing: false, query: 'architecture' },
};
const selected = selectAllVisible(searching);
expect(flattenSelection(selected.checked, selected.byId)).toEqual([IDS.architecture, IDS.marketing]);
expect([...selectNone(selected).checked]).toEqual([]);
});
it('returns save immediately for selected_roots and requires confirmation for all_accessible', () => {
const selectedRoots = toggleChecked(
buildInitialState({
tree: buildPickerTree(pages()),
existingRootPageIds: [],
currentCrawlMode: 'selected_roots',
}),
IDS.marketing,
1000,
);
expect(reducer(selectedRoots, 'save-request')).toEqual({
next: selectedRoots,
effect: 'save',
});
const allAccessible = {
...selectedRoots,
currentCrawlMode: 'all_accessible' as const,
};
const confirm = reducer(allAccessible, 'save-request');
expect(confirm).toEqual({
next: { ...allAccessible, pendingConfirm: 'mode-switch' },
effect: null,
});
expect(reducer(confirm.next, 'save-cancel')).toEqual({
next: { ...allAccessible, pendingConfirm: null },
effect: null,
});
expect(reducer(confirm.next, 'save-confirm')).toEqual({
next: { ...allAccessible, pendingConfirm: null },
effect: 'save',
});
});
it('blocks empty saves, updates search state, and quits without saving', () => {
const state = buildInitialState({
tree: buildPickerTree(pages()),
existingRootPageIds: [],
currentCrawlMode: 'selected_roots',
});
const blockedSave = reducer(state, 'save-request', 9000);
expect(blockedSave).toEqual({
next: {
...state,
transientHint: {
text: 'Select at least one page or press q to quit',
expiresAt: 9000 + TRANSIENT_HINT_DURATION_MS,
},
},
effect: null,
});
expect(
reducer(
reducer(reducer(state, 'search-start').next, { type: 'search-input', value: 'a' }).next,
'search-submit',
).next.search,
).toEqual({ editing: false, query: 'a' });
expect(reducer(state, 'quit')).toEqual({
next: state,
effect: 'quit-without-save',
});
});
it('clears transient hints only when their expiry time has passed', () => {
const state = buildInitialState({
tree: buildPickerTree(pages()),
existingRootPageIds: [],
currentCrawlMode: 'selected_roots',
});
const withHint = {
...state,
transientHint: {
text: 'Select at least one page or press q to quit',
expiresAt: 11500,
},
};
expect(clearExpiredTransientHint(withHint, 11499)).toBe(withHint);
expect(clearExpiredTransientHint(withHint, 11500)).toEqual({
...withHint,
transientHint: null,
});
expect(reducer(withHint, 'clear-transient-hint', 11501)).toEqual({
next: {
...withHint,
transientHint: null,
},
effect: null,
});
});
});

View file

@ -0,0 +1,529 @@
export interface NotionPickerPageInput {
id: string;
title?: string | null;
archived?: boolean;
parentId?: string | null;
}
interface NotionPickerNode {
id: string;
title: string;
archived: boolean;
parentId: string | null;
depth: number;
childIds: string[];
path: string;
}
export interface PickerState {
tree: NotionPickerNode[];
byId: Map<string, NotionPickerNode>;
expanded: Set<string>;
checked: Set<string>;
cursorId: string;
search: { editing: boolean; query: string };
pendingConfirm: 'mode-switch' | null;
preLoadWarnings: string[];
transientHint: { text: string; expiresAt: number } | null;
currentCrawlMode: 'all_accessible' | 'selected_roots';
}
export type PickerCommand =
| 'cursor-up'
| 'cursor-down'
| 'cursor-left'
| 'cursor-right'
| 'expand'
| 'collapse'
| 'expand-all'
| 'collapse-all'
| 'toggle-check'
| 'select-all-visible'
| 'select-none'
| 'clear-transient-hint'
| 'search-start'
| 'search-cancel'
| 'search-submit'
| 'search-backspace'
| { type: 'search-input'; value: string }
| 'save-request'
| 'save-confirm'
| 'save-cancel'
| 'quit';
type PickerEffect = null | 'save' | 'quit-without-save';
interface MutableNode {
id: string;
title: string;
archived: boolean;
parentId: string | null;
childIds: string[];
}
export const TRANSIENT_HINT_DURATION_MS = 2500;
const collator = new Intl.Collator('en', { sensitivity: 'base', numeric: true });
function normalizePageId(value: string): string {
const trimmed = value.trim();
const compact = trimmed.replace(/-/g, '');
if (/^[0-9a-fA-F]{32}$/.test(compact)) {
const lower = compact.toLowerCase();
return `${lower.slice(0, 8)}-${lower.slice(8, 12)}-${lower.slice(12, 16)}-${lower.slice(
16,
20,
)}-${lower.slice(20)}`;
}
return trimmed;
}
function titleValue(value: string | null | undefined): string {
const trimmed = value?.trim() ?? '';
return trimmed.length > 0 ? trimmed : 'Untitled';
}
function sortedNodeIds(ids: string[], nodes: Map<string, MutableNode | NotionPickerNode>): string[] {
return [...ids].sort((leftId, rightId) => {
const left = nodes.get(leftId);
const right = nodes.get(rightId);
const byTitle = collator.compare(left?.title ?? '', right?.title ?? '');
return byTitle === 0 ? leftId.localeCompare(rightId) : byTitle;
});
}
function cloneState(state: PickerState, patch: Partial<PickerState>): PickerState {
return { ...state, ...patch };
}
function transientHint(text: string, now: number): PickerState['transientHint'] {
return { text, expiresAt: now + TRANSIENT_HINT_DURATION_MS };
}
export function clearExpiredTransientHint(state: PickerState, now = Date.now()): PickerState {
if (!state.transientHint || state.transientHint.expiresAt > now) {
return state;
}
return cloneState(state, { transientHint: null });
}
function ancestorsOf(nodeId: string, byId: Map<string, NotionPickerNode>): string[] {
const ancestors: string[] = [];
let parentId = byId.get(nodeId)?.parentId ?? null;
const seen = new Set<string>();
while (parentId && !seen.has(parentId)) {
ancestors.push(parentId);
seen.add(parentId);
parentId = byId.get(parentId)?.parentId ?? null;
}
return ancestors;
}
function descendantsOf(nodeId: string, byId: Map<string, NotionPickerNode>): string[] {
const result: string[] = [];
const stack = [...(byId.get(nodeId)?.childIds ?? [])].reverse();
while (stack.length > 0) {
const id = stack.pop();
if (!id) {
continue;
}
result.push(id);
const node = byId.get(id);
if (node) {
stack.push(...[...node.childIds].reverse());
}
}
return result;
}
function matchingIds(state: PickerState): Set<string> {
const query = state.search.query.trim().toLocaleLowerCase();
if (!query) {
return new Set(state.tree.map((node) => node.id));
}
return new Set(
state.tree
.filter((node) => {
const title = node.title.toLocaleLowerCase();
const path = node.path.toLocaleLowerCase();
return title.includes(query) || path.includes(query);
})
.map((node) => node.id),
);
}
export function buildPickerTree(searchResults: NotionPickerPageInput[]): NotionPickerNode[] {
const nodes = new Map<string, MutableNode>();
for (const result of searchResults) {
const id = normalizePageId(result.id);
if (nodes.has(id)) {
continue;
}
nodes.set(id, {
id,
title: titleValue(result.title),
archived: result.archived === true,
parentId: result.parentId ? normalizePageId(result.parentId) : null,
childIds: [],
});
}
for (const node of nodes.values()) {
if (!node.parentId || node.parentId === node.id || !nodes.has(node.parentId)) {
node.parentId = null;
continue;
}
const seen = new Set([node.id]);
let cursor: string | null = node.parentId;
while (cursor) {
if (seen.has(cursor)) {
node.parentId = null;
break;
}
seen.add(cursor);
cursor = nodes.get(cursor)?.parentId ?? null;
}
}
for (const node of nodes.values()) {
node.childIds = [];
}
for (const node of nodes.values()) {
if (node.parentId) {
nodes.get(node.parentId)?.childIds.push(node.id);
}
}
for (const node of nodes.values()) {
node.childIds = sortedNodeIds(node.childIds, nodes);
}
const roots = sortedNodeIds(
[...nodes.values()].filter((node) => node.parentId === null).map((node) => node.id),
nodes,
);
const tree: NotionPickerNode[] = [];
function visit(nodeId: string, depth: number, pathPrefix: string[]): void {
const raw = nodes.get(nodeId);
if (!raw) {
return;
}
const path = [...pathPrefix, raw.title].join(' / ');
const node: NotionPickerNode = {
id: raw.id,
title: raw.title,
archived: raw.archived,
parentId: raw.parentId,
depth,
childIds: raw.childIds,
path,
};
tree.push(node);
for (const childId of raw.childIds) {
visit(childId, depth + 1, [...pathPrefix, raw.title]);
}
}
for (const rootId of roots) {
visit(rootId, 0, []);
}
return tree;
}
export function isAncestorChecked(nodeId: string, checked: Set<string>, byId: Map<string, NotionPickerNode>): boolean {
return ancestorsOf(nodeId, byId).some((ancestorId) => checked.has(ancestorId));
}
function checkedAncestor(nodeId: string, state: PickerState): NotionPickerNode | null {
for (const ancestorId of ancestorsOf(nodeId, state.byId)) {
if (state.checked.has(ancestorId)) {
return state.byId.get(ancestorId) ?? null;
}
}
return null;
}
export function canToggle(nodeId: string, state: PickerState): { ok: true } | { ok: false; reason: string } {
if (!state.byId.has(nodeId)) {
return { ok: false, reason: 'Page not found' };
}
const ancestor = checkedAncestor(nodeId, state);
if (ancestor) {
return { ok: false, reason: `Locked by '${ancestor.title}' - uncheck parent first` };
}
return { ok: true };
}
export function toggleChecked(state: PickerState, nodeId: string, now = Date.now()): PickerState {
const toggle = canToggle(nodeId, state);
if (!toggle.ok) {
return cloneState(state, {
transientHint: transientHint(toggle.reason, now),
});
}
const checked = new Set(state.checked);
if (checked.has(nodeId)) {
checked.delete(nodeId);
} else {
checked.add(nodeId);
for (const descendantId of descendantsOf(nodeId, state.byId)) {
checked.delete(descendantId);
}
}
return cloneState(state, { checked, transientHint: null });
}
export function flattenSelection(checked: Set<string>, byId: Map<string, NotionPickerNode>): string[] {
const result: string[] = [];
for (const node of byId.values()) {
if (checked.has(node.id) && !isAncestorChecked(node.id, checked, byId)) {
result.push(node.id);
}
}
return result;
}
export function filterTree(state: PickerState): { visibleIds: Set<string>; autoExpand: Set<string> } {
const matches = matchingIds(state);
if (state.search.query.trim().length === 0) {
return { visibleIds: matches, autoExpand: new Set() };
}
const visibleIds = new Set<string>();
const autoExpand = new Set<string>();
for (const matchId of matches) {
visibleIds.add(matchId);
for (const ancestorId of ancestorsOf(matchId, state.byId)) {
visibleIds.add(ancestorId);
autoExpand.add(ancestorId);
}
}
return { visibleIds, autoExpand };
}
export function visibleNodeIds(state: PickerState): string[] {
const { visibleIds, autoExpand } = filterTree(state);
const result: string[] = [];
const roots = state.tree.filter((node) => node.parentId === null).map((node) => node.id);
function visit(nodeId: string): void {
if (!visibleIds.has(nodeId)) {
return;
}
result.push(nodeId);
const node = state.byId.get(nodeId);
if (!node) {
return;
}
if (state.expanded.has(nodeId) || autoExpand.has(nodeId)) {
for (const childId of node.childIds) {
visit(childId);
}
}
}
for (const rootId of roots) {
visit(rootId);
}
return result;
}
export function selectAllVisible(state: PickerState): PickerState {
const candidates = state.search.query.trim().length > 0 ? matchingIds(state) : new Set(visibleNodeIds(state));
const checked = new Set(state.checked);
for (const node of state.tree) {
if (!candidates.has(node.id)) {
continue;
}
const hasCandidateAncestor = ancestorsOf(node.id, state.byId).some((ancestorId) => candidates.has(ancestorId));
if (!hasCandidateAncestor && !isAncestorChecked(node.id, checked, state.byId)) {
checked.add(node.id);
for (const descendantId of descendantsOf(node.id, state.byId)) {
checked.delete(descendantId);
}
}
}
return cloneState(state, {
checked: new Set(flattenSelection(checked, state.byId)),
transientHint: null,
});
}
export function selectNone(state: PickerState): PickerState {
return cloneState(state, { checked: new Set(), transientHint: null });
}
function setExpanded(state: PickerState, nodeId: string, value: boolean | 'toggle'): PickerState {
const expanded = new Set(state.expanded);
const nextValue = value === 'toggle' ? !expanded.has(nodeId) : value;
if (nextValue) {
expanded.add(nodeId);
} else {
expanded.delete(nodeId);
}
return cloneState(state, { expanded });
}
function expandPath(state: PickerState, nodeId: string): PickerState {
const expanded = new Set(state.expanded);
for (const ancestorId of ancestorsOf(nodeId, state.byId)) {
expanded.add(ancestorId);
}
return cloneState(state, { expanded });
}
export function moveCursor(state: PickerState, dir: 'up' | 'down' | 'left' | 'right'): PickerState {
const node = state.byId.get(state.cursorId);
if (!node) {
return state;
}
if (dir === 'left') {
if (node.childIds.length > 0 && state.expanded.has(node.id)) {
return setExpanded(state, node.id, false);
}
return node.parentId ? cloneState(state, { cursorId: node.parentId }) : state;
}
if (dir === 'right') {
if (node.childIds.length === 0) {
return state;
}
if (!state.expanded.has(node.id)) {
return setExpanded(state, node.id, true);
}
return cloneState(state, { cursorId: node.childIds[0] ?? node.id });
}
const ids = visibleNodeIds(state);
const index = ids.indexOf(state.cursorId);
if (index === -1) {
return ids[0] ? cloneState(state, { cursorId: ids[0] }) : state;
}
const nextIndex = dir === 'up' ? Math.max(0, index - 1) : Math.min(ids.length - 1, index + 1);
return cloneState(state, { cursorId: ids[nextIndex] ?? state.cursorId });
}
export function buildInitialState(args: {
tree: NotionPickerNode[];
existingRootPageIds: string[];
currentCrawlMode?: 'all_accessible' | 'selected_roots';
}): PickerState {
const byId = new Map(args.tree.map((node) => [node.id, node]));
const checked = new Set<string>();
let staleCount = 0;
for (const rawId of args.existingRootPageIds) {
const id = normalizePageId(rawId);
if (byId.has(id)) {
checked.add(id);
} else {
staleCount += 1;
}
}
const minimalChecked = new Set(flattenSelection(checked, byId));
const expanded = new Set<string>();
for (const checkedId of minimalChecked) {
for (const ancestorId of ancestorsOf(checkedId, byId)) {
expanded.add(ancestorId);
}
}
return {
tree: args.tree,
byId,
expanded,
checked: minimalChecked,
cursorId: args.tree[0]?.id ?? '',
search: { editing: false, query: '' },
pendingConfirm: null,
preLoadWarnings: staleCount > 0 ? [`${staleCount} stored root_page_ids no longer visible`] : [],
transientHint: null,
currentCrawlMode: args.currentCrawlMode ?? 'selected_roots',
};
}
export function reducer(state: PickerState, cmd: PickerCommand, now = Date.now()): { next: PickerState; effect: PickerEffect } {
if (state.pendingConfirm) {
if (cmd === 'save-confirm') {
return { next: cloneState(state, { pendingConfirm: null }), effect: 'save' };
}
if (cmd === 'save-cancel') {
return { next: cloneState(state, { pendingConfirm: null }), effect: null };
}
if (cmd === 'quit') {
return { next: state, effect: 'quit-without-save' };
}
return { next: state, effect: null };
}
switch (cmd) {
case 'cursor-up':
return { next: moveCursor(state, 'up'), effect: null };
case 'cursor-down':
return { next: moveCursor(state, 'down'), effect: null };
case 'cursor-left':
return { next: moveCursor(state, 'left'), effect: null };
case 'cursor-right':
return { next: moveCursor(state, 'right'), effect: null };
case 'expand':
return { next: setExpanded(state, state.cursorId, 'toggle'), effect: null };
case 'collapse':
return { next: setExpanded(state, state.cursorId, false), effect: null };
case 'expand-all':
return {
next: cloneState(state, {
expanded: new Set(state.tree.filter((node) => node.childIds.length > 0).map((node) => node.id)),
}),
effect: null,
};
case 'collapse-all':
return { next: cloneState(state, { expanded: new Set() }), effect: null };
case 'toggle-check':
return { next: toggleChecked(state, state.cursorId, now), effect: null };
case 'select-all-visible':
return { next: selectAllVisible(state), effect: null };
case 'select-none':
return { next: selectNone(state), effect: null };
case 'clear-transient-hint':
return { next: clearExpiredTransientHint(state, now), effect: null };
case 'search-start':
return { next: cloneState(state, { search: { ...state.search, editing: true } }), effect: null };
case 'search-cancel':
return { next: cloneState(state, { search: { editing: false, query: '' } }), effect: null };
case 'search-submit':
return { next: cloneState(state, { search: { ...state.search, editing: false } }), effect: null };
case 'search-backspace':
return {
next: cloneState(state, { search: { ...state.search, query: state.search.query.slice(0, -1) } }),
effect: null,
};
case 'save-request':
if (state.checked.size === 0) {
return {
next: cloneState(state, {
transientHint: transientHint('Select at least one page or press q to quit', now),
}),
effect: null,
};
}
if (state.currentCrawlMode === 'all_accessible') {
return { next: cloneState(state, { pendingConfirm: 'mode-switch' }), effect: null };
}
return { next: state, effect: 'save' };
case 'save-confirm':
return { next: state, effect: 'save' };
case 'save-cancel':
return { next: state, effect: null };
case 'quit':
return { next: state, effect: 'quit-without-save' };
default:
return { next: cloneState(state, { search: { ...state.search, query: state.search.query + cmd.value } }), effect: null };
}
}

View file

@ -0,0 +1,384 @@
/* @jsxImportSource react */
import { render as renderInkTest } from 'ink-testing-library';
import React, { act, type ReactNode } from 'react';
import { afterEach, describe, expect, it, vi } from 'vitest';
import { buildInitialState, buildPickerTree, type NotionPickerPageInput } from './connection-notion-tree.js';
import {
NotionPickerApp,
notionPickerCommandForInkInput,
renderNotionPickerTui,
resolveNotionPickerWidth,
sanitizeNotionPickerTuiError,
windowItems,
windowOffset,
type NotionPickerInkInstance,
type NotionPickerInkRenderOptions,
} from './connection-notion-tui.js';
const IDS = {
engineering: '11111111-1111-1111-1111-111111111111',
architecture: '22222222-2222-2222-2222-222222222222',
marketing: '33333333-3333-3333-3333-333333333333',
finance: '44444444-4444-4444-4444-444444444444',
ops: '55555555-5555-5555-5555-555555555555',
sales: '66666666-6666-6666-6666-666666666666',
support: '77777777-7777-7777-7777-777777777777',
product: '88888888-8888-8888-8888-888888888888',
design: '99999999-9999-9999-9999-999999999999',
};
function pages(): NotionPickerPageInput[] {
return [
{ id: IDS.engineering, title: 'Engineering Docs', archived: false, parentId: null },
{ id: IDS.architecture, title: 'Architecture', archived: false, parentId: IDS.engineering },
{ id: IDS.marketing, title: 'Marketing', archived: false, parentId: null },
];
}
function manyPages(): NotionPickerPageInput[] {
return [
{ id: IDS.engineering, title: 'Engineering Docs', archived: false, parentId: null },
{ id: IDS.architecture, title: 'Architecture', archived: false, parentId: IDS.engineering },
{ id: IDS.marketing, title: 'Marketing', archived: false, parentId: null },
{ id: IDS.finance, title: 'Finance', archived: false, parentId: null },
{ id: IDS.ops, title: 'Operations', archived: false, parentId: null },
{ id: IDS.sales, title: 'Sales', archived: false, parentId: null },
{ id: IDS.support, title: 'Support', archived: false, parentId: null },
{ id: IDS.product, title: 'Product', archived: false, parentId: null },
{ id: IDS.design, title: 'Design', archived: false, parentId: null },
];
}
function state(mode: 'all_accessible' | 'selected_roots' = 'selected_roots') {
return buildInitialState({
tree: buildPickerTree(pages()),
existingRootPageIds: [],
currentCrawlMode: mode,
});
}
async function waitForInkInput(): Promise<void> {
await new Promise((resolve) => setTimeout(resolve, 10));
}
function fakeInkInstance(): NotionPickerInkInstance {
return {
rerender: vi.fn(),
unmount: vi.fn(),
waitUntilExit: vi.fn(async () => undefined),
};
}
function normalizeFrameWrap(frame: string | undefined): string {
return frame?.replace(/\n/g, ' ') ?? '';
}
afterEach(() => {
vi.useRealTimers();
});
describe('notionPickerCommandForInkInput', () => {
it('maps browse, search, and confirm input to reducer commands', () => {
expect(notionPickerCommandForInkInput('', { downArrow: true }, state().search, null)).toBe('cursor-down');
expect(notionPickerCommandForInkInput('', { upArrow: true }, state().search, null)).toBe('cursor-up');
expect(notionPickerCommandForInkInput('', { rightArrow: true }, state().search, null)).toBe('cursor-right');
expect(notionPickerCommandForInkInput('', { leftArrow: true }, state().search, null)).toBe('cursor-left');
expect(notionPickerCommandForInkInput(' ', {}, state().search, null)).toBe('toggle-check');
expect(notionPickerCommandForInkInput('/', {}, state().search, null)).toBe('search-start');
expect(notionPickerCommandForInkInput('a', {}, state().search, null)).toBe('select-all-visible');
expect(notionPickerCommandForInkInput('n', {}, state().search, null)).toBe('select-none');
expect(notionPickerCommandForInkInput('s', {}, state().search, null)).toBe('save-request');
expect(notionPickerCommandForInkInput('q', {}, state().search, null)).toBe('quit');
expect(notionPickerCommandForInkInput('c', { ctrl: true }, state().search, null)).toBe('quit');
expect(notionPickerCommandForInkInput('x', {}, { editing: true, query: '' }, null)).toEqual({
type: 'search-input',
value: 'x',
});
expect(notionPickerCommandForInkInput('', { backspace: true }, { editing: true, query: 'x' }, null)).toBe(
'search-backspace',
);
expect(notionPickerCommandForInkInput('', { return: true }, { editing: true, query: 'x' }, null)).toBe(
'search-submit',
);
expect(notionPickerCommandForInkInput('', { escape: true }, { editing: true, query: 'x' }, null)).toBe(
'search-cancel',
);
expect(notionPickerCommandForInkInput('y', {}, state().search, 'mode-switch')).toBe('save-confirm');
expect(notionPickerCommandForInkInput('', { return: true }, state().search, 'mode-switch')).toBe('save-confirm');
expect(notionPickerCommandForInkInput('n', {}, state().search, 'mode-switch')).toBe('save-cancel');
});
});
describe('window helpers', () => {
it('centers the selected row and returns the visible slice', () => {
expect(windowOffset(20, 10, 5)).toBe(8);
expect(windowItems(['a', 'b', 'c', 'd', 'e'], 3, 3)).toEqual({ items: ['c', 'd', 'e'], offset: 2 });
});
it('clamps picker width to the design rule', () => {
expect(resolveNotionPickerWidth(200)).toBe(120);
expect(resolveNotionPickerWidth(100)).toBe(96);
expect(resolveNotionPickerWidth(50)).toBe(60);
expect(resolveNotionPickerWidth(undefined)).toBe(96);
});
});
describe('NotionPickerApp', () => {
it('renders spec banners, row glyphs, search visibility, and hint text', () => {
const initialState = {
...state('all_accessible'),
preLoadWarnings: ['1 stored root_page_ids no longer visible'],
};
const { lastFrame } = renderInkTest(
<NotionPickerApp
initialState={initialState}
connectionId="notion-main"
workspaceLabel="Design Workspace"
cappedAtCount={5000}
currentCrawlMode="all_accessible"
terminalRows={24}
terminalWidth={100}
onExit={vi.fn()}
/>,
);
const frame = lastFrame() ?? '';
expect(frame).toContain('Notion pages visible to integration "Design Workspace"');
expect(frame).toContain('5000-page cap reached - some pages not shown');
expect(frame).toContain('1 stored root_page_ids no longer visible - they will be removed if you save');
expect(frame).toContain('▸ [ ] Engineering Docs ▸ (1)');
expect(frame).toContain(' [ ] Marketing');
expect(frame).not.toContain('Search ready: -');
expect(frame).toContain('space toggle · enter expand · / search · a all · n none · s save & exit · q quit');
});
it('renders partial discovery warnings without stale-root save suffix', () => {
const initialState = {
...state(),
preLoadWarnings: ['Notion search stopped early: rate limit after first page'],
};
const { lastFrame } = renderInkTest(
<NotionPickerApp
initialState={initialState}
connectionId="notion-main"
workspaceLabel="Design Workspace"
cappedAtCount={null}
currentCrawlMode="selected_roots"
terminalRows={24}
terminalWidth={100}
onExit={vi.fn()}
/>,
);
const frame = lastFrame() ?? '';
expect(frame).toContain('Notion search stopped early: rate limit after first page');
expect(frame).not.toContain(
'Notion search stopped early: rate limit after first page - they will be removed if you save',
);
});
it('renders checked parents and locked descendants with the locked design glyphs', () => {
const initialState = {
...state(),
checked: new Set([IDS.engineering]),
expanded: new Set([IDS.engineering]),
};
const { lastFrame } = renderInkTest(
<NotionPickerApp
initialState={initialState}
connectionId="notion-main"
workspaceLabel="Design Workspace"
cappedAtCount={null}
currentCrawlMode="selected_roots"
terminalRows={24}
terminalWidth={100}
onExit={vi.fn()}
/>,
);
const frame = lastFrame() ?? '';
expect(frame).toContain('▸ [×] Engineering Docs ▾');
expect(frame).toContain(' [~] Architecture');
});
it('supports keyboard selection, all_accessible confirmation, and save callback', async () => {
const onExit = vi.fn();
const { stdin, lastFrame } = renderInkTest(
<NotionPickerApp
initialState={state('all_accessible')}
connectionId="notion-main"
workspaceLabel="Design Workspace"
cappedAtCount={null}
currentCrawlMode="all_accessible"
terminalRows={24}
terminalWidth={100}
onExit={onExit}
/>,
);
stdin.write(' ');
await waitForInkInput();
expect(lastFrame()).toContain('[×] Engineering Docs');
stdin.write('s');
await waitForInkInput();
expect(normalizeFrameWrap(lastFrame())).toContain(
'Save will switch crawl_mode all_accessible -> selected_roots and limit ingest to 1 selected page. [y] confirm [esc] back',
);
stdin.write('y');
await waitForInkInput();
expect(onExit).toHaveBeenCalledWith({ kind: 'save', rootPageIds: [IDS.engineering] });
});
it('removes transient hints after their expiry time', async () => {
vi.useFakeTimers();
const onExit = vi.fn();
const { stdin, lastFrame } = renderInkTest(
<NotionPickerApp
initialState={state()}
connectionId="notion-main"
workspaceLabel="Design Workspace"
cappedAtCount={null}
currentCrawlMode="selected_roots"
terminalRows={24}
terminalWidth={100}
onExit={onExit}
/>,
);
await act(async () => {
stdin.write('s');
await vi.advanceTimersByTimeAsync(10);
});
expect(lastFrame()).toContain('Select at least one page or press q to quit');
await act(async () => {
await vi.advanceTimersByTimeAsync(2500);
});
expect(lastFrame()).not.toContain('Select at least one page or press q to quit');
expect(onExit).not.toHaveBeenCalled();
});
it('renders row-window overflow indicators when the visible list is clipped', async () => {
const onExit = vi.fn();
const initialState = buildInitialState({
tree: buildPickerTree(manyPages()),
existingRootPageIds: [],
currentCrawlMode: 'selected_roots',
});
initialState.expanded = new Set([IDS.engineering]);
const { stdin, lastFrame } = renderInkTest(
<NotionPickerApp
initialState={initialState}
connectionId="notion-main"
workspaceLabel="Design Workspace"
cappedAtCount={null}
currentCrawlMode="selected_roots"
terminalRows={13}
terminalWidth={100}
onExit={onExit}
/>,
);
expect(lastFrame()).toContain('↓ 4 more');
stdin.write('\u001B[B');
stdin.write('\u001B[B');
stdin.write('\u001B[B');
stdin.write('\u001B[B');
await waitForInkInput();
const frame = lastFrame() ?? '';
expect(frame).toContain('↑ ');
expect(frame).toContain('↓ ');
expect(onExit).not.toHaveBeenCalled();
});
it('returns quit without saving', async () => {
const onExit = vi.fn();
const { stdin } = renderInkTest(
<NotionPickerApp
initialState={state()}
connectionId="notion-main"
workspaceLabel="Design Workspace"
cappedAtCount={null}
currentCrawlMode="selected_roots"
terminalRows={24}
terminalWidth={100}
onExit={onExit}
/>,
);
stdin.write('q');
await waitForInkInput();
expect(onExit).toHaveBeenCalledWith({ kind: 'quit' });
});
});
describe('renderNotionPickerTui', () => {
it('returns the app result from the Ink runtime', async () => {
const io = {
stdin: { isTTY: true, setRawMode: vi.fn() },
stdout: { isTTY: true, columns: 100, rows: 24, write: vi.fn() },
stderr: { write: vi.fn() },
};
const renderInk = vi.fn((_tree: ReactNode, _options: NotionPickerInkRenderOptions) => fakeInkInstance());
await expect(
renderNotionPickerTui(
{
initialState: state(),
connectionId: 'notion-main',
workspaceLabel: 'Design Workspace',
cappedAtCount: null,
currentCrawlMode: 'selected_roots',
},
io,
{ renderInk },
),
).resolves.toEqual({ kind: 'quit' });
expect(renderInk).toHaveBeenCalledOnce();
});
it('sanitizes render errors and tells the user to use no-input mode', async () => {
expect(sanitizeNotionPickerTuiError(new Error('token=secret https://api.notion.com/v1/search'))).toBe(
'[redacted] [redacted-url]',
);
});
it('falls back to quit with a scripted-mode hint when Ink cannot initialize', async () => {
let stderr = '';
const io = {
stdin: { isTTY: false, setRawMode: vi.fn() },
stdout: { isTTY: false, columns: 100, rows: 24, write: vi.fn() },
stderr: {
write(chunk: string) {
stderr += chunk;
},
},
};
await expect(
renderNotionPickerTui(
{
initialState: state(),
connectionId: 'notion-main',
workspaceLabel: 'Design Workspace',
cappedAtCount: null,
currentCrawlMode: 'selected_roots',
},
io,
{
renderInk: vi.fn(() => {
throw new Error('token=secret');
}),
},
),
).resolves.toEqual({ kind: 'quit' });
expect(stderr).toContain('Use --no-input --root-page-id <UUID> for scripted mode');
expect(stderr).not.toContain('secret');
});
});

View file

@ -0,0 +1,338 @@
/* @jsxImportSource react */
import { Box, Text, render as renderInkRuntime, useApp, useInput } from 'ink';
import React, { type ReactNode, useEffect, useMemo, useRef, useState } from 'react';
import {
filterTree,
flattenSelection,
isAncestorChecked,
reducer,
visibleNodeIds,
type PickerCommand,
type PickerState,
} from './connection-notion-tree.js';
import type { KloCliIo } from '../index.js';
const COLOR_THEME = {
text: 'white',
muted: 'gray',
active: 'cyan',
warning: 'yellow',
} as const;
const NO_COLOR_THEME = {
text: 'white',
muted: 'white',
active: 'white',
warning: 'white',
} as const;
type NotionPickerTheme = Record<keyof typeof COLOR_THEME, string>;
export interface NotionPickerTuiIo extends KloCliIo {
stdin?: { isTTY?: boolean; setRawMode?(value: boolean): void };
stdout: KloCliIo['stdout'] & { isTTY?: boolean; columns?: number; rows?: number };
}
interface InkKey {
leftArrow?: boolean;
rightArrow?: boolean;
upArrow?: boolean;
downArrow?: boolean;
return?: boolean;
escape?: boolean;
ctrl?: boolean;
backspace?: boolean;
delete?: boolean;
}
export type PickerRenderResult = { kind: 'save'; rootPageIds: string[] } | { kind: 'quit' };
export interface PickerRenderInput {
initialState: PickerState;
connectionId: string;
workspaceLabel: string;
cappedAtCount: number | null;
currentCrawlMode: 'all_accessible' | 'selected_roots';
}
interface NotionPickerAppProps extends PickerRenderInput {
terminalRows?: number;
terminalWidth?: number;
env?: NodeJS.ProcessEnv;
onExit(result: PickerRenderResult): void;
}
export interface NotionPickerInkInstance {
rerender(tree: ReactNode): void;
unmount(): void;
waitUntilExit(): Promise<void>;
}
export interface NotionPickerInkRenderOptions {
stdin?: NotionPickerTuiIo['stdin'];
stdout: NotionPickerTuiIo['stdout'];
stderr: NotionPickerTuiIo['stderr'];
exitOnCtrlC: boolean;
patchConsole: boolean;
maxFps: number;
alternateScreen: boolean;
}
function resolveTheme(env: NodeJS.ProcessEnv = process.env): NotionPickerTheme {
return env.NO_COLOR || env.TERM === 'dumb' ? NO_COLOR_THEME : COLOR_THEME;
}
export function resolveNotionPickerWidth(columns: number | undefined): number {
const resolvedColumns = columns ?? 100;
return Math.max(60, Math.min(120, resolvedColumns - 4));
}
function staleWarningText(warning: string): string {
return warning.includes('stored root_page_ids no longer visible')
? `${warning} - they will be removed if you save`
: warning;
}
function selectedPageCountText(count: number): string {
return `${count} selected ${count === 1 ? 'page' : 'pages'}`;
}
function rowMatchesSearch(state: PickerState, nodeId: string): boolean {
const query = state.search.query.trim().toLocaleLowerCase();
if (!query) {
return false;
}
const node = state.byId.get(nodeId);
if (!node) {
return false;
}
return node.title.toLocaleLowerCase().includes(query) || node.path.toLocaleLowerCase().includes(query);
}
export function sanitizeNotionPickerTuiError(error: unknown): string {
const message = error instanceof Error ? error.message : String(error);
return message
.replace(/[a-z][a-z0-9+.-]*:\/\/[^\s]+/gi, '[redacted-url]')
.replace(/\b(api[_-]?key|password|token|secret)=\S+/gi, '[redacted]');
}
export function windowOffset(count: number, selected: number, visible: number): number {
if (count <= visible) return 0;
return Math.max(0, Math.min(count - visible, selected - Math.floor(visible / 2)));
}
export function windowItems<T>(items: T[], selected: number, visible: number): { items: T[]; offset: number } {
const offset = windowOffset(items.length, selected, visible);
return { items: items.slice(offset, offset + visible), offset };
}
function truncateText(value: string, width: number): string {
if (value.length <= width) return value;
if (width <= 3) return value.slice(0, width);
return `${value.slice(0, width - 3)}...`;
}
export function notionPickerCommandForInkInput(
input: string,
key: InkKey,
search: PickerState['search'],
pendingConfirm: PickerState['pendingConfirm'],
): PickerCommand | null {
if (pendingConfirm) {
if (input === 'y' || key.return) return 'save-confirm';
if (input === 'n' || key.escape) return 'save-cancel';
if (key.ctrl === true && input === 'c') return 'quit';
return null;
}
if (search.editing) {
if (key.escape) return 'search-cancel';
if (key.return) return 'search-submit';
if (key.backspace || key.delete) return 'search-backspace';
if (key.downArrow) return 'cursor-down';
if (key.upArrow) return 'cursor-up';
if (input.length === 1 && input >= ' ' && input !== '\u007f') return { type: 'search-input', value: input };
return null;
}
if (key.ctrl === true && input === 'c') return 'quit';
if (key.upArrow) return 'cursor-up';
if (key.downArrow) return 'cursor-down';
if (key.leftArrow) return 'cursor-left';
if (key.rightArrow) return 'cursor-right';
if (key.return) return 'expand';
if (input === ' ') return 'toggle-check';
if (input === '/') return 'search-start';
if (input === 'a') return 'select-all-visible';
if (input === 'n') return 'select-none';
if (input === 's') return 'save-request';
if (input === 'q' || key.escape) return 'quit';
return null;
}
function PickerRow(props: { state: PickerState; nodeId: string; width: number; theme: NotionPickerTheme }): ReactNode {
const node = props.state.byId.get(props.nodeId);
if (!node) return null;
const focused = props.state.cursorId === node.id;
const locked = isAncestorChecked(node.id, props.state.checked, props.state.byId);
const checked = props.state.checked.has(node.id);
const glyph = locked ? '[~]' : checked ? '[×]' : '[ ]';
const children =
node.childIds.length > 0 ? (props.state.expanded.has(node.id) ? ' ▾' : ` ▸ (${node.childIds.length})`) : '';
const prefix = `${focused ? '▸' : ' '} ${glyph} ${' '.repeat(node.depth * 2)}`;
const color = focused ? props.theme.active : locked || node.archived ? props.theme.muted : props.theme.text;
const title = truncateText(`${node.title}${children}`, Math.max(10, props.width - prefix.length));
const inverse = rowMatchesSearch(props.state, node.id);
return (
<Text color={color} strikethrough={node.archived}>
{prefix}
<Text inverse={inverse}>{title}</Text>
</Text>
);
}
export function NotionPickerApp(props: NotionPickerAppProps): ReactNode {
const app = useApp();
const [state, setState] = useState(props.initialState);
const stateRef = useRef(state);
const theme = useMemo(() => resolveTheme(props.env), [props.env]);
const visibleIds = visibleNodeIds(state);
const selectedIndex = Math.max(0, visibleIds.indexOf(state.cursorId));
const reservedRows = state.pendingConfirm === 'mode-switch' ? 9 : 8;
const visibleRows = Math.max(5, Math.min(20, (props.terminalRows ?? 24) - reservedRows));
const rows = windowItems(visibleIds, selectedIndex, visibleRows);
const hiddenAbove = rows.offset;
const hiddenBelow = Math.max(0, visibleIds.length - rows.offset - rows.items.length);
const searchMatchCount = filterTree(state).visibleIds.size;
const width = resolveNotionPickerWidth(props.terminalWidth);
const showSearch = state.search.editing || state.search.query.trim().length > 0;
const selectedCount = flattenSelection(state.checked, state.byId).length;
stateRef.current = state;
useEffect(() => {
const hint = state.transientHint;
if (!hint) {
return;
}
const clearHint = () => {
setState((current) => {
const { next } = reducer(current, 'clear-transient-hint');
stateRef.current = next;
return next;
});
};
const delay = hint.expiresAt - Date.now();
if (delay <= 0) {
clearHint();
return;
}
const timeout = setTimeout(clearHint, delay);
return () => clearTimeout(timeout);
}, [state.transientHint?.expiresAt]);
useInput((input, key) => {
const command = notionPickerCommandForInkInput(input, key, stateRef.current.search, stateRef.current.pendingConfirm);
if (!command) {
return;
}
const { next, effect } = reducer(stateRef.current, command);
stateRef.current = next;
setState(next);
if (effect === 'save') {
props.onExit({ kind: 'save', rootPageIds: flattenSelection(next.checked, next.byId) });
app.exit();
return;
}
if (effect === 'quit-without-save') {
props.onExit({ kind: 'quit' });
app.exit();
}
});
return (
<Box flexDirection="column">
<Text color={theme.active}>Notion pages visible to integration "{props.workspaceLabel}"</Text>
{props.cappedAtCount ? <Text color={theme.warning}>{props.cappedAtCount}-page cap reached - some pages not shown</Text> : null}
{state.preLoadWarnings.map((warning) => (
<Text key={warning} color={theme.warning}>
{staleWarningText(warning)}
</Text>
))}
{showSearch ? (
<Text color={theme.muted}>
/ {state.search.query}
{state.search.editing ? '█' : ''} ({searchMatchCount} matches)
</Text>
) : null}
<Box flexDirection="column">
{hiddenAbove > 0 ? <Text color={theme.muted}> {hiddenAbove} more</Text> : null}
{rows.items.map((nodeId) => (
<PickerRow key={nodeId} state={state} nodeId={nodeId} width={width} theme={theme} />
))}
{hiddenBelow > 0 ? <Text color={theme.muted}> {hiddenBelow} more</Text> : null}
</Box>
{state.pendingConfirm === 'mode-switch' ? (
<Text color={theme.warning}>
Save will switch crawl_mode all_accessible -&gt; selected_roots and limit ingest to{' '}
{selectedPageCountText(selectedCount)}. [y] confirm [esc] back
</Text>
) : null}
{state.transientHint ? <Text color={theme.warning}>{state.transientHint.text}</Text> : null}
<Text color={theme.muted}>space toggle · enter expand · / search · a all · n none · s save &amp; exit · q quit</Text>
</Box>
);
}
function renderInk(tree: ReactNode, options: NotionPickerInkRenderOptions): NotionPickerInkInstance {
return renderInkRuntime(tree, {
stdin: options.stdin as NodeJS.ReadStream | undefined,
stdout: options.stdout as NodeJS.WriteStream,
stderr: options.stderr as NodeJS.WriteStream,
exitOnCtrlC: options.exitOnCtrlC,
patchConsole: options.patchConsole,
maxFps: options.maxFps,
alternateScreen: options.alternateScreen,
}) as NotionPickerInkInstance;
}
export async function renderNotionPickerTui(
input: PickerRenderInput,
io: NotionPickerTuiIo,
options: { renderInk?: (tree: ReactNode, options: NotionPickerInkRenderOptions) => NotionPickerInkInstance } = {},
): Promise<PickerRenderResult> {
let result: PickerRenderResult = { kind: 'quit' };
let instance: NotionPickerInkInstance | null = null;
try {
instance = (options.renderInk ?? renderInk)(
<NotionPickerApp
{...input}
terminalRows={(io.stdout as { rows?: number }).rows ?? process.stdout.rows ?? 24}
terminalWidth={io.stdout.columns ?? process.stdout.columns}
onExit={(next) => {
result = next;
instance?.unmount();
}}
/>,
{
stdin: io.stdin,
stdout: io.stdout,
stderr: io.stderr,
exitOnCtrlC: false,
patchConsole: false,
maxFps: 30,
alternateScreen: true,
},
);
await instance.waitUntilExit();
instance.unmount();
return result;
} catch (error) {
io.stderr.write(
`Notion picker requires a TTY. Use --no-input --root-page-id <UUID> for scripted mode. ${sanitizeNotionPickerTuiError(error)}\n`,
);
return { kind: 'quit' };
}
}

View file

@ -0,0 +1,466 @@
import { mkdtemp, readFile, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import {
initKloProject,
loadKloProject,
serializeKloProjectConfig,
type KloProjectConfig,
} from '@klo/context/project';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import {
applyNotionPickerWriteback,
discoverNotionPickerPages,
notionPickerPageFromSearchResult,
normalizeNotionPageId,
resolveNotionWorkspaceLabel,
runKloConnectionNotion,
type NotionPickerApi,
type PickerRenderInput,
type PickerRenderResult,
} from './connection-notion.js';
function makeIo() {
let stdout = '';
let stderr = '';
return {
io: {
stdout: {
write: (chunk: string) => {
stdout += chunk;
},
},
stderr: {
write: (chunk: string) => {
stderr += chunk;
},
},
},
stdout: () => stdout,
stderr: () => stderr,
};
}
type FakeNotionSearchPage = Record<string, unknown> & { id: string; object: 'page' };
const PAGE_IDS = {
engineering: '11111111-1111-1111-1111-111111111111',
architecture: '22222222-2222-2222-2222-222222222222',
stale: '99999999-9999-9999-9999-999999999999',
};
function notionPage(id: string, title: string, parentId: string | null = null): FakeNotionSearchPage {
return {
object: 'page',
id,
archived: false,
parent: parentId ? { type: 'page_id', page_id: parentId } : { type: 'workspace', workspace: true },
properties: {
title: {
type: 'title',
title: [{ plain_text: title }],
},
},
};
}
function fakeNotionApi(pages: FakeNotionSearchPage[]): NotionPickerApi {
return {
search: vi.fn(async (_filterValue, startCursor) => {
if (startCursor === 'page-2') {
return { results: pages.slice(2), hasMore: false, nextCursor: null };
}
return {
results: pages.slice(0, 2),
hasMore: pages.length > 2,
nextCursor: pages.length > 2 ? 'page-2' : null,
};
}),
retrieveBotUser: vi.fn(async () => ({ name: 'Notion bot', bot: { workspace_name: 'Design Workspace' } })),
};
}
describe('normalizeNotionPageId', () => {
it('accepts dashed and compact UUIDs', () => {
expect(normalizeNotionPageId('11111111222233334444555555555555')).toBe(
'11111111-2222-3333-4444-555555555555',
);
expect(normalizeNotionPageId('AAAAAAAA-BBBB-CCCC-DDDD-EEEEEEEEEEEE')).toBe(
'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee',
);
});
});
describe('runKloConnectionNotion', () => {
let tempDir: string;
beforeEach(async () => {
tempDir = await mkdtemp(join(tmpdir(), 'klo-cli-notion-pick-'));
});
afterEach(async () => {
await rm(tempDir, { recursive: true, force: true });
});
async function writeProjectConfig(projectDir: string, config: KloProjectConfig): Promise<void> {
const project = await loadKloProject({ projectDir });
await project.fileStore.writeFile(
'klo.yaml',
serializeKloProjectConfig(config),
'klo',
'klo@example.com',
'seed test config',
);
}
it('rejects unsafe connection ids before loading a project', async () => {
const io = makeIo();
const loadProject = vi.fn(async () => {
throw new Error('loadProject should not be called');
});
await expect(
runKloConnectionNotion(
{
command: 'pick',
projectDir: '/tmp/project',
connectionId: '../evil',
mode: 'interactive',
},
io.io,
{ loadProject },
),
).resolves.toBe(1);
expect(loadProject).not.toHaveBeenCalled();
expect(io.stderr()).toContain('Unsafe connection id: ../evil');
});
it('writes selected root_page_ids while preserving every other Notion connection field', async () => {
const projectDir = join(tempDir, 'project');
const initialized = await initKloProject({ projectDir, projectName: 'warehouse' });
await writeProjectConfig(projectDir, {
...initialized.config,
connections: {
'notion-main': {
driver: 'notion',
auth_token_ref: 'env:NOTION_TOKEN',
crawl_mode: 'all_accessible',
root_page_ids: ['99999999-9999-9999-9999-999999999999'],
root_database_ids: ['database-1'],
root_data_source_ids: ['data-source-1'],
max_pages_per_run: 12,
max_knowledge_creates_per_run: 2,
max_knowledge_updates_per_run: 7,
last_successful_cursor: '{"phase":"all_accessible_pages","cursor":"cursor-1"}',
unknown_future_field: 'keep-me',
},
},
});
const io = makeIo();
await expect(
runKloConnectionNotion(
{
command: 'pick',
projectDir,
connectionId: 'notion-main',
mode: 'non-interactive',
rootPageIds: [
'11111111-2222-3333-4444-555555555555',
'66666666-7777-8888-9999-aaaaaaaaaaaa',
],
},
io.io,
),
).resolves.toBe(0);
const yaml = await readFile(join(projectDir, 'klo.yaml'), 'utf-8');
expect(yaml).toContain('crawl_mode: selected_roots');
expect(yaml).toContain('root_page_ids:');
expect(yaml).toContain('11111111-2222-3333-4444-555555555555');
expect(yaml).toContain('66666666-7777-8888-9999-aaaaaaaaaaaa');
expect(yaml).toContain('root_database_ids:');
expect(yaml).toContain('database-1');
expect(yaml).toContain('root_data_source_ids:');
expect(yaml).toContain('data-source-1');
expect(yaml).toContain('last_successful_cursor: \'{"phase":"all_accessible_pages","cursor":"cursor-1"}\'');
expect(yaml).toContain('unknown_future_field: keep-me');
expect(io.stdout()).toContain('Connection: notion-main');
expect(io.stdout()).toContain('rootPageIds: 2');
expect(io.stdout()).toContain('crawlMode: selected_roots');
});
it('rejects empty writeback, missing connections, and non-Notion connections', async () => {
const projectDir = join(tempDir, 'project');
const initialized = await initKloProject({ projectDir, projectName: 'warehouse' });
await writeProjectConfig(projectDir, {
...initialized.config,
connections: {
warehouse: {
driver: 'postgres',
url: 'env:DATABASE_URL',
readonly: true,
},
},
});
const project = await loadKloProject({ projectDir });
await expect(applyNotionPickerWriteback(project, 'warehouse', [])).rejects.toThrow(
'connection notion pick requires at least one root page id',
);
await expect(
applyNotionPickerWriteback(project, 'missing', ['11111111-2222-3333-4444-555555555555']),
).rejects.toThrow('Connection "missing" not found');
await expect(
applyNotionPickerWriteback(project, 'warehouse', ['11111111-2222-3333-4444-555555555555']),
).rejects.toThrow('Connection "warehouse" is not a Notion connection');
});
it('extracts picker page inputs from Notion search results', () => {
expect(notionPickerPageFromSearchResult(notionPage(PAGE_IDS.architecture, 'Architecture', PAGE_IDS.engineering)))
.toEqual({
id: PAGE_IDS.architecture,
title: 'Architecture',
archived: false,
parentId: PAGE_IDS.engineering,
});
expect(
notionPickerPageFromSearchResult({
object: 'page',
id: PAGE_IDS.engineering.replaceAll('-', ''),
archived: true,
parent: { type: 'workspace', workspace: true },
properties: {},
}),
).toEqual({
id: PAGE_IDS.engineering,
title: 'Untitled',
archived: true,
parentId: null,
});
});
it('discovers visible pages up to the cap and reports cap state', async () => {
const api = fakeNotionApi([
notionPage(PAGE_IDS.engineering, 'Engineering'),
notionPage(PAGE_IDS.architecture, 'Architecture', PAGE_IDS.engineering),
notionPage('33333333-3333-3333-3333-333333333333', 'Onboarding', PAGE_IDS.engineering),
]);
await expect(discoverNotionPickerPages(api, { cap: 2 })).resolves.toEqual({
pages: [
{ id: PAGE_IDS.engineering, title: 'Engineering', archived: false, parentId: null },
{ id: PAGE_IDS.architecture, title: 'Architecture', archived: false, parentId: PAGE_IDS.engineering },
],
cappedAtCount: 2,
warnings: [],
});
expect(api.search).toHaveBeenCalledTimes(1);
});
it('keeps partial discovery results when Notion search fails after at least one page', async () => {
const api: NotionPickerApi = {
search: vi
.fn()
.mockResolvedValueOnce({
results: [notionPage(PAGE_IDS.engineering, 'Engineering')],
hasMore: true,
nextCursor: 'cursor-2',
})
.mockRejectedValueOnce(new Error('rate limit after first page')),
retrieveBotUser: vi.fn(async () => ({ name: 'Notion bot' })),
};
await expect(discoverNotionPickerPages(api)).resolves.toEqual({
pages: [{ id: PAGE_IDS.engineering, title: 'Engineering', archived: false, parentId: null }],
cappedAtCount: null,
warnings: ['Notion search stopped early: rate limit after first page'],
});
});
it('uses the Notion workspace name when available and falls back to the connection id', async () => {
await expect(resolveNotionWorkspaceLabel(fakeNotionApi([]), 'notion-main')).resolves.toBe('Design Workspace');
await expect(
resolveNotionWorkspaceLabel(
{
search: vi.fn(),
retrieveBotUser: vi.fn(async () => {
throw new Error('users.me unavailable');
}),
},
'notion-main',
),
).resolves.toBe('notion-main');
});
it('runs interactive discovery, warns about stale roots, renders the TUI, and saves selected roots', async () => {
const projectDir = join(tempDir, 'project');
const initialized = await initKloProject({ projectDir, projectName: 'warehouse' });
await writeProjectConfig(projectDir, {
...initialized.config,
connections: {
'notion-main': {
driver: 'notion',
auth_token_ref: 'env:NOTION_TOKEN',
crawl_mode: 'all_accessible',
root_page_ids: [PAGE_IDS.stale],
root_database_ids: ['database-1'],
root_data_source_ids: ['data-source-1'],
max_pages_per_run: 12,
max_knowledge_creates_per_run: 2,
max_knowledge_updates_per_run: 7,
last_successful_cursor: null,
},
},
});
const api = fakeNotionApi([
notionPage(PAGE_IDS.engineering, 'Engineering'),
notionPage(PAGE_IDS.architecture, 'Architecture', PAGE_IDS.engineering),
]);
const renderPicker = vi.fn(async (input): Promise<PickerRenderResult> => {
expect(input.connectionId).toBe('notion-main');
expect(input.workspaceLabel).toBe('Design Workspace');
expect(input.currentCrawlMode).toBe('all_accessible');
expect(input.cappedAtCount).toBeNull();
expect(input.initialState.preLoadWarnings).toEqual(['1 stored root_page_ids no longer visible']);
return { kind: 'save', rootPageIds: [PAGE_IDS.engineering] };
});
const io = makeIo();
await expect(
runKloConnectionNotion(
{
command: 'pick',
projectDir,
connectionId: 'notion-main',
mode: 'interactive',
},
io.io,
{
env: { NOTION_TOKEN: 'ntn_test_token' },
createNotionApi: vi.fn(() => api),
renderPicker,
},
),
).resolves.toBe(0);
const yaml = await readFile(join(projectDir, 'klo.yaml'), 'utf-8');
expect(yaml).toContain('crawl_mode: selected_roots');
expect(yaml).toContain(PAGE_IDS.engineering);
expect(yaml).not.toContain(PAGE_IDS.stale);
expect(io.stderr()).toContain('1 stored root_page_ids no longer visible');
expect(io.stdout()).toContain('Connection: notion-main');
expect(io.stdout()).toContain('rootPageIds: 1');
});
it('passes partial-discovery warnings into the TUI banner state', async () => {
const projectDir = join(tempDir, 'project');
const initialized = await initKloProject({ projectDir, projectName: 'warehouse' });
await writeProjectConfig(projectDir, {
...initialized.config,
connections: {
'notion-main': {
driver: 'notion',
auth_token_ref: 'env:NOTION_TOKEN',
crawl_mode: 'selected_roots',
root_page_ids: [PAGE_IDS.engineering],
root_database_ids: [],
root_data_source_ids: [],
max_pages_per_run: 12,
max_knowledge_creates_per_run: 2,
max_knowledge_updates_per_run: 7,
last_successful_cursor: null,
},
},
});
const api: NotionPickerApi = {
search: vi
.fn()
.mockResolvedValueOnce({
results: [notionPage(PAGE_IDS.engineering, 'Engineering')],
hasMore: true,
nextCursor: 'cursor-2',
})
.mockRejectedValueOnce(new Error('rate limit after first page')),
retrieveBotUser: vi.fn(async () => ({ name: 'Notion bot', bot: { workspace_name: 'Design Workspace' } })),
};
let renderInput: PickerRenderInput | undefined;
const renderPicker = vi.fn(async (input: PickerRenderInput): Promise<PickerRenderResult> => {
renderInput = input;
return { kind: 'quit' };
});
const io = makeIo();
await expect(
runKloConnectionNotion(
{
command: 'pick',
projectDir,
connectionId: 'notion-main',
mode: 'interactive',
},
io.io,
{
env: { NOTION_TOKEN: 'ntn_test_token' },
createNotionApi: vi.fn(() => api),
renderPicker,
},
),
).resolves.toBe(0);
expect(renderPicker).toHaveBeenCalledOnce();
if (!renderInput) {
throw new Error('renderPicker was not called');
}
expect(renderInput.initialState.preLoadWarnings).toEqual(['Notion search stopped early: rate limit after first page']);
expect(renderInput.initialState.tree.map((node) => node.title)).toEqual(['Engineering']);
expect(io.stderr()).toContain('Notion search stopped early: rate limit after first page');
expect(io.stdout()).toContain('No changes saved.');
});
it('quits interactive mode without writing when the TUI returns quit', async () => {
const projectDir = join(tempDir, 'project');
const initialized = await initKloProject({ projectDir, projectName: 'warehouse' });
await writeProjectConfig(projectDir, {
...initialized.config,
connections: {
'notion-main': {
driver: 'notion',
auth_token_ref: 'env:NOTION_TOKEN',
crawl_mode: 'selected_roots',
root_page_ids: [PAGE_IDS.engineering],
root_database_ids: [],
root_data_source_ids: [],
max_pages_per_run: 12,
max_knowledge_creates_per_run: 2,
max_knowledge_updates_per_run: 7,
last_successful_cursor: null,
},
},
});
const before = await readFile(join(projectDir, 'klo.yaml'), 'utf-8');
const io = makeIo();
await expect(
runKloConnectionNotion(
{
command: 'pick',
projectDir,
connectionId: 'notion-main',
mode: 'interactive',
},
io.io,
{
env: { NOTION_TOKEN: 'ntn_test_token' },
createNotionApi: vi.fn(() => fakeNotionApi([notionPage(PAGE_IDS.engineering, 'Engineering')])),
renderPicker: vi.fn(async (): Promise<PickerRenderResult> => ({ kind: 'quit' })),
},
),
).resolves.toBe(0);
await expect(readFile(join(projectDir, 'klo.yaml'), 'utf-8')).resolves.toBe(before);
expect(io.stdout()).toContain('No changes saved.');
});
});

View file

@ -0,0 +1,278 @@
import { parseNotionConnectionConfig, resolveNotionAuthToken } from '@klo/context/connections';
import { type NotionApi, type NotionBotInfo, NotionClient } from '@klo/context/ingest';
import {
type KloLocalProject,
type KloProjectConnectionConfig,
loadKloProject,
serializeKloProjectConfig,
} from '@klo/context/project';
import type { KloCliIo } from '../index.js';
import { profileMark } from '../startup-profile.js';
import { buildInitialState, buildPickerTree, type NotionPickerPageInput } from './connection-notion-tree.js';
import {
type NotionPickerTuiIo,
type PickerRenderInput,
type PickerRenderResult,
renderNotionPickerTui,
} from './connection-notion-tui.js';
profileMark('module:commands/connection-notion');
export type KloConnectionNotionArgs =
| {
command: 'pick';
projectDir: string;
connectionId: string;
mode: 'interactive';
}
| {
command: 'pick';
projectDir: string;
connectionId: string;
mode: 'non-interactive';
rootPageIds: string[];
};
export type NotionPickerApi = Pick<NotionApi, 'search' | 'retrieveBotUser'>;
export type { PickerRenderInput, PickerRenderResult };
interface KloConnectionNotionDeps {
env?: Record<string, string | undefined>;
loadProject?: typeof loadKloProject;
createNotionApi?: (authToken: string) => NotionPickerApi;
renderPicker?: (input: PickerRenderInput, io: NotionPickerTuiIo) => Promise<PickerRenderResult>;
}
const NOTION_PICKER_PAGE_CAP = 5000;
function assertSafeConnectionId(connectionId: string): void {
if (!/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(connectionId)) {
throw new Error(`Unsafe connection id: ${connectionId}`);
}
}
export function normalizeNotionPageId(value: string): string {
const trimmed = value.trim();
const compact = trimmed.includes('-') ? trimmed.replace(/-/g, '') : trimmed;
if (!/^[0-9a-fA-F]{32}$/.test(compact)) {
throw new Error(`Invalid Notion page UUID: ${value}`);
}
const lower = compact.toLowerCase();
return `${lower.slice(0, 8)}-${lower.slice(8, 12)}-${lower.slice(12, 16)}-${lower.slice(16, 20)}-${lower.slice(20)}`;
}
function recordValue(value: unknown): Record<string, unknown> | null {
return typeof value === 'object' && value !== null && !Array.isArray(value)
? (value as Record<string, unknown>)
: null;
}
function extractTitleFromNotionPage(page: Record<string, unknown>): string {
const properties = recordValue(page.properties);
if (!properties) {
return 'Untitled';
}
for (const property of Object.values(properties)) {
const value = recordValue(property);
if (!value || value.type !== 'title' || !Array.isArray(value.title)) {
continue;
}
const text = value.title
.map((part) => {
const richText = recordValue(part);
return typeof richText?.plain_text === 'string' ? richText.plain_text : '';
})
.join('')
.trim();
if (text.length > 0) {
return text;
}
}
return 'Untitled';
}
function extractParentPageId(page: Record<string, unknown>): string | null {
const parent = recordValue(page.parent);
if (!parent || parent.type !== 'page_id' || typeof parent.page_id !== 'string') {
return null;
}
return normalizeNotionPageId(parent.page_id);
}
export function notionPickerPageFromSearchResult(result: Record<string, unknown>): NotionPickerPageInput {
const id = typeof result.id === 'string' ? normalizeNotionPageId(result.id) : '';
if (!id) {
throw new Error('Notion page search result is missing id');
}
return {
id,
title: extractTitleFromNotionPage(result),
archived: result.archived === true,
parentId: extractParentPageId(result),
};
}
export async function discoverNotionPickerPages(
api: NotionPickerApi,
options: { cap?: number } = {},
): Promise<{ pages: NotionPickerPageInput[]; cappedAtCount: number | null; warnings: string[] }> {
const cap = options.cap ?? NOTION_PICKER_PAGE_CAP;
const pages: NotionPickerPageInput[] = [];
const warnings: string[] = [];
let cursor: string | null | undefined = null;
while (pages.length < cap) {
let response: Awaited<ReturnType<NotionPickerApi['search']>>;
try {
response = await api.search('page', cursor, Math.min(100, cap - pages.length));
} catch (error) {
if (pages.length === 0) {
throw error;
}
const message = error instanceof Error ? error.message : String(error);
warnings.push(`Notion search stopped early: ${message}`);
return { pages, cappedAtCount: null, warnings };
}
for (const result of response.results) {
pages.push(notionPickerPageFromSearchResult(result));
if (pages.length >= cap) {
break;
}
}
if (!response.hasMore || !response.nextCursor || pages.length >= cap) {
return {
pages,
cappedAtCount: response.hasMore ? cap : null,
warnings,
};
}
cursor = response.nextCursor;
}
return { pages, cappedAtCount: cap, warnings };
}
export async function resolveNotionWorkspaceLabel(api: NotionPickerApi, connectionId: string): Promise<string> {
try {
const bot = (await api.retrieveBotUser()) as NotionBotInfo;
const workspaceName = typeof bot.bot?.workspace_name === 'string' ? bot.bot.workspace_name.trim() : '';
if (workspaceName.length > 0) {
return workspaceName;
}
const name = typeof bot.name === 'string' ? bot.name.trim() : '';
return name.length > 0 ? name : connectionId;
} catch {
return connectionId;
}
}
function notionConnection(project: KloLocalProject, connectionId: string): KloProjectConnectionConfig {
const connection = project.config.connections[connectionId];
if (!connection) {
throw new Error(`Connection "${connectionId}" not found`);
}
if (connection.driver !== 'notion') {
throw new Error(`Connection "${connectionId}" is not a Notion connection`);
}
return connection;
}
export async function applyNotionPickerWriteback(
project: KloLocalProject,
connectionId: string,
rootPageIds: string[],
): Promise<void> {
if (rootPageIds.length === 0) {
throw new Error('connection notion pick requires at least one root page id');
}
const existing = notionConnection(project, connectionId);
const nextConfig = {
...project.config,
connections: {
...project.config.connections,
[connectionId]: {
...existing,
crawl_mode: 'selected_roots',
root_page_ids: rootPageIds,
},
},
};
await project.fileStore.writeFile(
'klo.yaml',
serializeKloProjectConfig(nextConfig),
'klo',
'klo@example.com',
`Pick Notion roots: ${connectionId} (${rootPageIds.length} pages)`,
);
}
export async function runKloConnectionNotion(
args: KloConnectionNotionArgs,
io: KloCliIo = process,
deps: KloConnectionNotionDeps = {},
): Promise<number> {
try {
assertSafeConnectionId(args.connectionId);
const loadProject = deps.loadProject ?? loadKloProject;
if (args.mode === 'interactive') {
const project = await loadProject({ projectDir: args.projectDir });
const rawConnection = notionConnection(project, args.connectionId);
const notion = parseNotionConnectionConfig(rawConnection);
const authToken = await resolveNotionAuthToken(notion.auth_token_ref, { env: deps.env });
const api = deps.createNotionApi ? deps.createNotionApi(authToken) : new NotionClient(authToken);
const discovery = await discoverNotionPickerPages(api);
const tree = buildPickerTree(discovery.pages);
const initialState = buildInitialState({
tree,
existingRootPageIds: notion.root_page_ids,
currentCrawlMode: notion.crawl_mode,
});
const preLoadWarnings = [...discovery.warnings, ...initialState.preLoadWarnings];
const renderState =
preLoadWarnings.length > 0
? {
...initialState,
preLoadWarnings,
}
: initialState;
for (const warning of preLoadWarnings) {
io.stderr.write(`${warning}\n`);
}
const workspaceLabel = await resolveNotionWorkspaceLabel(api, args.connectionId);
const result = await (deps.renderPicker ?? renderNotionPickerTui)(
{
initialState: renderState,
connectionId: args.connectionId,
workspaceLabel,
cappedAtCount: discovery.cappedAtCount,
currentCrawlMode: notion.crawl_mode,
},
io as NotionPickerTuiIo,
);
if (result.kind === 'quit') {
io.stdout.write('No changes saved.\n');
return 0;
}
await applyNotionPickerWriteback(project, args.connectionId, result.rootPageIds);
io.stdout.write(`Connection: ${args.connectionId}\n`);
io.stdout.write(`rootPageIds: ${result.rootPageIds.length}\n`);
io.stdout.write('crawlMode: selected_roots\n');
return 0;
}
const project = await loadProject({ projectDir: args.projectDir });
await applyNotionPickerWriteback(project, args.connectionId, args.rootPageIds);
io.stdout.write(`Connection: ${args.connectionId}\n`);
io.stdout.write(`rootPageIds: ${args.rootPageIds.length}\n`);
io.stdout.write('crawlMode: selected_roots\n');
return 0;
} catch (error) {
io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
return 1;
}
}

View file

@ -0,0 +1,26 @@
import { describe, expect, it } from 'vitest';
import { resolveDemoCommandOptions } from './demo-commands.js';
describe('resolveDemoCommandOptions', () => {
it('lets parent --no-input override a child default from optsWithGlobals', () => {
const rootCommand = {
opts: () => ({}),
};
const setupCommand = {
parent: rootCommand,
opts: () => ({ input: false }),
getOptionValueSource: (name: string) => (name === 'input' ? 'cli' : undefined),
};
const demoCommand = {
parent: setupCommand,
opts: () => ({ input: true, mode: 'seeded' }),
optsWithGlobals: () => ({ input: true, mode: 'seeded' }),
getOptionValueSource: (name: string) => (name === 'input' ? 'default' : name === 'mode' ? 'default' : undefined),
};
expect(resolveDemoCommandOptions<{ input: boolean; mode: string }>(demoCommand)).toEqual({
input: false,
mode: 'seeded',
});
});
});

View file

@ -0,0 +1,273 @@
import { type Command, Option } from '@commander-js/extra-typings';
import {
type CommandWithGlobalOptions,
type KloCliCommandContext,
resolveCommandProjectDirOverride,
} from '../cli-program.js';
import {
type KloDemoArgs,
type KloDemoInputMode,
type KloDemoMode,
type KloDemoOutputMode,
} from '../demo.js';
import { defaultDemoProjectDir } from '../demo-assets.js';
import { resolveProjectDir } from '../project-dir.js';
import { profileMark } from '../startup-profile.js';
profileMark('module:commands/demo-commands');
interface DemoOptions {
plain?: boolean;
json?: boolean;
input?: boolean;
projectDir?: string;
}
function demoOutputMode(options: { plain?: boolean; json?: boolean }): KloDemoOutputMode {
if (options.json === true) {
return 'json';
}
if (options.plain === true) {
return 'plain';
}
return 'viz';
}
function demoDoctorOutputMode(options: { json?: boolean }): 'plain' | 'json' {
return options.json === true ? 'json' : 'plain';
}
function demoInspectOutputMode(options: { plain?: boolean; json?: boolean }): KloDemoOutputMode {
if (options.json === true) {
return 'json';
}
return 'plain';
}
function demoInputMode(options: { input?: boolean }): { inputMode?: KloDemoInputMode } {
return options.input === false ? { inputMode: 'disabled' } : {};
}
function demoProjectDir(options: { projectDir?: string }, command: CommandWithGlobalOptions): string {
return resolveProjectDir(
options.projectDir ?? resolveCommandProjectDirOverride(command),
defaultDemoProjectDir(),
);
}
type CommandOptionSourceReader = {
getOptionValueSource?: (name: string) => string | undefined;
parent?: unknown;
};
function inheritedOptionSource(command: CommandOptionSourceReader, key: string): string | undefined {
let current = command.parent as (CommandOptionSourceReader & { opts?: () => Record<string, unknown> }) | undefined;
while (current) {
const source = current.getOptionValueSource?.(key);
if (source !== undefined) {
return source;
}
current = current.parent as (CommandOptionSourceReader & { opts?: () => Record<string, unknown> }) | undefined;
}
return undefined;
}
function definedOptions(
options: Record<string, unknown>,
inherited: Record<string, unknown> = {},
command?: CommandOptionSourceReader,
): Record<string, unknown> {
return Object.fromEntries(
Object.entries(options).filter(([key, value]) => {
if (value === undefined) return false;
if (key === 'input' && value === true && inherited.input === false) return false;
if (
key === 'mode' &&
command?.getOptionValueSource?.(key) === 'default' &&
inherited[key] !== undefined &&
inherited[key] !== value &&
inheritedOptionSource(command, key) === 'cli'
) {
return false;
}
return true;
}),
);
}
export function resolveDemoCommandOptions<T>(command: { opts: () => T; optsWithGlobals?: () => T; parent?: unknown }): T {
const chain: Array<{ opts?: () => Record<string, unknown>; parent?: unknown }> = [];
let current = command.parent as { opts?: () => Record<string, unknown>; parent?: unknown } | undefined;
while (current) {
chain.unshift(current);
current = current.parent as { opts?: () => Record<string, unknown>; parent?: unknown } | undefined;
}
const inherited = Object.assign({}, ...chain.map((parent) => definedOptions(parent.opts?.() ?? {})));
if (command.optsWithGlobals) {
const withGlobals = {
...inherited,
...definedOptions(command.optsWithGlobals() as Record<string, unknown>, inherited, command),
};
return {
...withGlobals,
...definedOptions(command.opts() as Record<string, unknown>, withGlobals, command),
} as T;
}
return { ...inherited, ...definedOptions(command.opts() as Record<string, unknown>, inherited, command) } as T;
}
async function runDemoArgs(context: KloCliCommandContext, args: KloDemoArgs): Promise<void> {
const runner = context.deps.demo ?? (await import('../demo.js')).runKloDemo;
context.setExitCode(await runner(args, context.io));
}
export function registerDemoCommands(
program: Command,
context: KloCliCommandContext,
options: { description?: string } = {},
): void {
const demo = program
.command('demo')
.description(options.description ?? 'Run the pre-seeded KLO demo or a full LLM-backed demo')
.addOption(
new Option('--mode <mode>', 'Demo mode: seeded (default), replay, or full')
.choices(['seeded', 'replay', 'full'])
.default('seeded'),
)
.option('--project-dir <path>', 'Demo project directory')
.addOption(new Option('--plain', 'Print plain text output instead of the visual demo').conflicts('json'))
.addOption(new Option('--json', 'Print JSON output').conflicts('plain'))
.option('--no-input', 'Disable interactive terminal input')
.showHelpAfterError()
.action(async (options: { mode: 'seeded' | 'replay' | 'full' } & DemoOptions, command) => {
const resolvedOptions = resolveDemoCommandOptions<typeof options>(command);
await runDemoArgs(context, {
command: resolvedOptions.mode,
projectDir: demoProjectDir(resolvedOptions, command),
outputMode: demoOutputMode(resolvedOptions),
...demoInputMode(resolvedOptions),
});
});
demo
.command('init')
.description('Initialize the packaged demo project')
.option('--project-dir <path>', 'Demo project directory')
.option('--force', 'Recreate an existing demo project', false)
.option('--no-input', 'Disable interactive terminal input')
.action(async (_options, command: { opts: () => { projectDir?: string; force?: boolean; input?: boolean } }) => {
const options = resolveDemoCommandOptions(command);
await runDemoArgs(context, {
command: 'init',
projectDir: demoProjectDir(options, command),
force: options.force === true,
...demoInputMode(options),
});
});
demo
.command('reset')
.description('Reset the packaged demo project')
.option('--project-dir <path>', 'Demo project directory')
.option('--force', 'Recreate the demo project without prompting', false)
.option('--no-input', 'Disable interactive terminal input')
.action(async (_options, command: { opts: () => { projectDir?: string; force?: boolean; input?: boolean } }) => {
const options = resolveDemoCommandOptions(command);
await runDemoArgs(context, {
command: 'reset',
projectDir: demoProjectDir(options, command),
force: options.force === true,
...demoInputMode(options),
});
});
demo
.command('replay')
.description('Replay the packaged demo memory-flow')
.option('--project-dir <path>', 'Demo project directory')
.addOption(new Option('--plain', 'Print plain text output instead of the visual demo').conflicts('json'))
.addOption(new Option('--json', 'Print JSON output').conflicts('plain'))
.option('--no-input', 'Disable interactive terminal input')
.action(async (_options, command: { opts: () => DemoOptions }) => {
const options = resolveDemoCommandOptions(command);
await runDemoArgs(context, {
command: 'replay',
projectDir: demoProjectDir(options, command),
outputMode: demoOutputMode(options),
...demoInputMode(options),
});
});
demo
.command('scan')
.description('Run the packaged demo scan')
.option('--project-dir <path>', 'Demo project directory')
.option('--no-input', 'Disable interactive terminal input')
.action(async (_options, command: { opts: () => { projectDir?: string; input?: boolean } }) => {
const options = resolveDemoCommandOptions(command);
await runDemoArgs(context, {
command: 'scan',
projectDir: demoProjectDir(options, command),
...demoInputMode(options),
});
});
demo
.command('inspect')
.description('Inspect packaged demo outputs')
.option('--project-dir <path>', 'Demo project directory')
.addOption(new Option('--plain', 'Print plain text output').conflicts('json'))
.addOption(new Option('--json', 'Print JSON output').conflicts('plain'))
.option('--no-input', 'Disable interactive terminal input')
.action(async (_options, command: { opts: () => DemoOptions }) => {
const options = resolveDemoCommandOptions(command);
await runDemoArgs(context, {
command: 'inspect',
projectDir: demoProjectDir(options, command),
outputMode: demoInspectOutputMode(options),
...demoInputMode(options),
});
});
demo
.command('doctor')
.description('Check packaged demo readiness')
.option('--project-dir <path>', 'Demo project directory')
.addOption(new Option('--plain', 'Print plain text output').conflicts('json'))
.addOption(new Option('--json', 'Print JSON output').conflicts('plain'))
.option('--no-input', 'Disable interactive terminal input')
.action(async (_options, command: { opts: () => DemoOptions }) => {
const options = resolveDemoCommandOptions(command);
await runDemoArgs(context, {
command: 'doctor',
projectDir: demoProjectDir(options, command),
outputMode: demoDoctorOutputMode(options),
...demoInputMode(options),
});
});
demo
.command('ingest')
.description('Run packaged demo ingest')
.addOption(
new Option('--mode <mode>', 'Demo ingest mode: full or seeded')
.choices(['full', 'seeded'])
.default('full'),
)
.option('--project-dir <path>', 'Demo project directory')
.addOption(new Option('--plain', 'Print plain text output instead of the visual demo').conflicts('json'))
.addOption(new Option('--json', 'Print JSON output').conflicts('plain'))
.option('--no-input', 'Disable interactive terminal input')
.action(async (_options, command: { opts: () => { mode: KloDemoMode } & DemoOptions }) => {
const options = resolveDemoCommandOptions(command);
await runDemoArgs(context, {
command: 'ingest',
mode: options.mode,
projectDir: demoProjectDir(options, command),
outputMode: demoOutputMode(options),
...demoInputMode(options),
});
});
}

View file

@ -0,0 +1,53 @@
import type { Command } from '@commander-js/extra-typings';
import { type CommandWithGlobalOptions, type KloCliCommandContext, resolveCommandProjectDir } from '../cli-program.js';
import type { KloDoctorArgs } from '../doctor.js';
import { profileMark } from '../startup-profile.js';
profileMark('module:commands/doctor-commands');
function outputMode(options: { json?: boolean }): 'plain' | 'json' {
return options.json === true ? 'json' : 'plain';
}
function inputMode(options: { input?: boolean }): { inputMode?: 'disabled' } {
return options.input === false ? { inputMode: 'disabled' } : {};
}
async function runDoctorArgs(context: KloCliCommandContext, args: KloDoctorArgs): Promise<void> {
const runner = context.deps.doctor ?? (await import('../doctor.js')).runKloDoctor;
context.setExitCode(await runner(args, context.io));
}
export function registerDoctorCommands(program: Command, context: KloCliCommandContext): void {
const doctor = program
.command('doctor')
.description('Check KLO setup, project, and demo readiness')
.option('--json', 'Print JSON output', false)
.option('--no-input', 'Disable interactive terminal input')
.action(async (options: { json?: boolean; input?: boolean }, command) => {
await runDoctorArgs(context, {
command: 'project',
projectDir: resolveCommandProjectDir(command),
outputMode: outputMode(options),
...inputMode(options),
});
});
doctor
.command('setup')
.description('Check KLO install, build, and local runtime readiness')
.option('--json', 'Print JSON output', false)
.option('--no-input', 'Disable interactive terminal input')
.action(
async (
_options: { json?: boolean; input?: boolean },
command: CommandWithGlobalOptions,
) => {
const options = (command.optsWithGlobals ? command.optsWithGlobals() : command.opts()) as {
json?: boolean;
input?: boolean;
};
await runDoctorArgs(context, { command: 'setup', outputMode: outputMode(options), ...inputMode(options) });
},
);
}

View file

@ -0,0 +1,171 @@
import { resolve } from 'node:path';
import { type Command, Option } from '@commander-js/extra-typings';
import { type KloCliCommandContext, type OutputModeOptions, resolveCommandProjectDir } from '../cli-program.js';
import type { KloCliDeps, KloCliIo } from '../index.js';
import type { KloIngestArgs, KloIngestOutputMode } from '../ingest.js';
import { profileMark } from '../startup-profile.js';
profileMark('module:commands/ingest-commands');
interface IngestCommandOptions {
runIngestWithProgress: (
args: KloIngestArgs,
io: KloCliIo,
deps: KloCliDeps,
defaultRunIngest: (args: KloIngestArgs, io: KloCliIo) => Promise<number>,
) => Promise<number>;
}
function outputMode(options: OutputModeOptions): KloIngestOutputMode {
if (options.json === true) {
return 'json';
}
if (options.viz === true) {
return 'viz';
}
return 'plain';
}
function watchOutputMode(options: OutputModeOptions): KloIngestOutputMode {
if (options.json === true) {
return 'json';
}
if (options.plain === true) {
return 'plain';
}
return 'viz';
}
function inputMode(options: OutputModeOptions): Pick<KloIngestArgs, 'inputMode'> {
return options.input === false ? { inputMode: 'disabled' } : {};
}
async function runIngestArgs(
context: KloCliCommandContext,
args: KloIngestArgs,
options: IngestCommandOptions,
): Promise<void> {
const { runKloIngest } = await import('../ingest.js');
context.setExitCode(await options.runIngestWithProgress(args, context.io, context.deps, runKloIngest));
}
export function registerIngestCommands(
program: Command,
context: KloCliCommandContext,
commandOptions: IngestCommandOptions,
): void {
const ingest = program
.command('ingest')
.description('Run or inspect local ingest memory-flow output')
.showHelpAfterError();
ingest.hook('preAction', (_thisCommand, actionCommand) => {
context.writeDebug?.('ingest', actionCommand);
});
ingest
.command('run')
.description('Run local ingest for one configured connection and source adapter')
.requiredOption('--connection-id <connectionId>', 'KLO connection id')
.requiredOption('--adapter <adapter>', 'Ingest source adapter name')
.option('--source-dir <path>', 'Directory containing source files')
.option('--database-introspection-url <url>', 'Daemon URL for live-database introspection')
.option('--debug-llm-request-file <path>', 'Write sanitized LLM request structure to a JSONL file')
.option('--report-file <path>', 'Unsupported for ingest run; use ingest status/watch instead')
.addOption(new Option('--plain', 'Print plain text output').conflicts(['json', 'viz']))
.addOption(new Option('--json', 'Print JSON output').conflicts(['plain', 'viz']))
.addOption(new Option('--viz', 'Render memory-flow TUI output').conflicts(['plain', 'json']))
.option('--no-input', 'Disable interactive terminal input for visualization')
.action(async (options, command) => {
if (options.reportFile) {
throw new Error('--report-file is only supported for ingest status/watch');
}
await runIngestArgs(
context,
{
command: 'run',
projectDir: resolveCommandProjectDir(command),
connectionId: options.connectionId,
adapter: options.adapter,
sourceDir: options.sourceDir ? resolve(options.sourceDir) : undefined,
databaseIntrospectionUrl: options.databaseIntrospectionUrl || undefined,
...(options.debugLlmRequestFile ? { debugLlmRequestFile: resolve(options.debugLlmRequestFile) } : {}),
outputMode: outputMode(options),
...inputMode(options),
},
commandOptions,
);
});
ingest
.command('status')
.description('Print status for the latest or selected stored local ingest run or report file')
.argument('[runId]', 'Local ingest run id, report id, run id, or job id')
.option('--report-file <path>', 'Bundle ingest report JSON file to render')
.addOption(new Option('--plain', 'Print plain text output').conflicts(['json', 'viz']))
.addOption(new Option('--json', 'Print JSON output').conflicts(['plain', 'viz']))
.addOption(new Option('--viz', 'Render memory-flow TUI output').conflicts(['plain', 'json']))
.option('--no-input', 'Disable interactive terminal input for visualization')
.action(async (runId: string | undefined, options, command) => {
await runIngestArgs(
context,
{
command: 'status',
projectDir: resolveCommandProjectDir(command),
...(runId ? { runId } : {}),
...(options.reportFile ? { reportFile: resolve(options.reportFile) } : {}),
outputMode: outputMode(options),
...inputMode(options),
},
commandOptions,
);
});
ingest
.command('watch')
.description('Open the latest or selected stored ingest visual report')
.argument('[runId]', 'Local ingest run id, report id, run id, or job id')
.option('--report-file <path>', 'Bundle ingest report JSON file to render')
.addOption(new Option('--plain', 'Print plain text output').conflicts(['json', 'viz']))
.addOption(new Option('--json', 'Print JSON output').conflicts(['plain', 'viz']))
.addOption(new Option('--viz', 'Render memory-flow TUI output').conflicts(['plain', 'json']))
.option('--no-input', 'Disable interactive terminal input for visualization')
.action(async (runId: string | undefined, options, command) => {
await runIngestArgs(
context,
{
command: 'watch',
projectDir: resolveCommandProjectDir(command),
...(runId ? { runId } : {}),
...(options.reportFile ? { reportFile: resolve(options.reportFile) } : {}),
outputMode: watchOutputMode(options),
...inputMode(options),
},
commandOptions,
);
});
ingest
.command('replay')
.description('Replay a stored ingest run or bundle report through memory-flow output')
.argument('<runId>', 'Local ingest run id, report id, run id, or job id')
.option('--report-file <path>', 'Bundle ingest report JSON file to render')
.addOption(new Option('--plain', 'Print plain text output').conflicts(['json', 'viz']))
.addOption(new Option('--json', 'Print JSON output').conflicts(['plain', 'viz']))
.addOption(new Option('--viz', 'Render memory-flow TUI output').conflicts(['plain', 'json']))
.option('--no-input', 'Disable interactive terminal input for visualization')
.action(async (runId: string, options, command) => {
await runIngestArgs(
context,
{
command: 'replay',
projectDir: resolveCommandProjectDir(command),
runId,
...(options.reportFile ? { reportFile: resolve(options.reportFile) } : {}),
outputMode: outputMode(options),
...inputMode(options),
},
commandOptions,
);
});
}

View file

@ -0,0 +1,90 @@
import { type Command, Option } from '@commander-js/extra-typings';
import { collectOption, type KloCliCommandContext, resolveCommandProjectDir } from '../cli-program.js';
import { wikiWriteCommandSchema } from '../command-schemas.js';
import type { KloKnowledgeArgs } from '../knowledge.js';
import { profileMark } from '../startup-profile.js';
profileMark('module:commands/knowledge-commands');
async function runKnowledgeArgs(context: KloCliCommandContext, args: KloKnowledgeArgs): Promise<void> {
const runner = context.deps.knowledge ?? (await import('../knowledge.js')).runKloKnowledge;
context.setExitCode(await runner(args, context.io));
}
export function registerWikiCommands(program: Command, context: KloCliCommandContext): void {
const wiki = program
.command('wiki')
.description('List, read, search, or write local wiki pages')
.showHelpAfterError()
.addHelpText(
'after',
'\nProject directory defaults to KLO_PROJECT_DIR when set, otherwise the current working directory.\n',
);
wiki
.command('list')
.description('List local wiki pages')
.option('--user-id <id>', 'Local user id', 'local')
.action(async (options: { userId: string }, command) => {
await runKnowledgeArgs(context, {
command: 'list',
projectDir: resolveCommandProjectDir(command),
userId: options.userId,
});
});
wiki
.command('read')
.description('Read one local wiki page')
.argument('<key>', 'Wiki page key')
.option('--user-id <id>', 'Local user id', 'local')
.action(async (key: string, options: { userId: string }, command) => {
await runKnowledgeArgs(context, {
command: 'read',
projectDir: resolveCommandProjectDir(command),
key,
userId: options.userId,
});
});
wiki
.command('search')
.description('Search local wiki pages')
.argument('<query>', 'Search query')
.option('--user-id <id>', 'Local user id', 'local')
.action(async (query: string, options: { userId: string }, command) => {
await runKnowledgeArgs(context, {
command: 'search',
projectDir: resolveCommandProjectDir(command),
query,
userId: options.userId,
});
});
wiki
.command('write')
.description('Write one local wiki page')
.argument('<key>', 'Wiki page key')
.option('--user-id <id>', 'Local user id', 'local')
.addOption(new Option('--scope <scope>', 'global or user').choices(['global', 'user']).default('global'))
.requiredOption('--summary <summary>', 'Wiki summary')
.requiredOption('--content <content>', 'Wiki content')
.option('--tag <tag>', 'Wiki tag; repeatable', collectOption, [])
.option('--ref <ref>', 'Wiki ref; repeatable', collectOption, [])
.option('--sl-ref <ref>', 'Semantic-layer ref; repeatable', collectOption, [])
.action(async (key: string, options, command) => {
const args = wikiWriteCommandSchema.parse({
command: 'write',
projectDir: resolveCommandProjectDir(command),
key,
scope: options.scope === 'user' ? 'USER' : 'GLOBAL',
userId: options.userId,
summary: options.summary,
content: options.content,
tags: options.tag,
refs: options.ref,
slRefs: options.slRef,
});
await runKnowledgeArgs(context, args);
});
}

View file

@ -0,0 +1,109 @@
import { InvalidArgumentError, type Command } from '@commander-js/extra-typings';
import { type KloCliCommandContext, resolveCommandProjectDir } from '../cli-program.js';
import { publicIngestReadCommandSchema, publicIngestRunCommandSchema } from '../command-schemas.js';
import type { KloPublicIngestArgs, KloPublicIngestInputMode } from '../public-ingest.js';
import { profileMark } from '../startup-profile.js';
profileMark('module:commands/public-ingest-commands');
interface PublicIngestOptions {
all?: boolean;
json?: boolean;
input?: boolean;
}
function inputMode(options: { input?: boolean }): KloPublicIngestInputMode {
return options.input === false ? 'disabled' : 'auto';
}
async function runPublicIngestArgs(context: KloCliCommandContext, args: KloPublicIngestArgs): Promise<void> {
const runner = context.deps.publicIngest ?? (await import('../public-ingest.js')).runKloPublicIngest;
context.setExitCode(await runner(args, context.io));
}
function parsePublicIngestConnectionId(value: string): string {
if (value === 'run') {
throw new InvalidArgumentError('run is reserved; use klo dev ingest run for low-level adapter syntax');
}
return value;
}
export function registerPublicIngestCommands(program: Command, context: KloCliCommandContext): void {
const ingest = program
.command('ingest')
.description('Build and refresh KLO context from configured sources')
.usage('[options] [connectionId]')
.argument('[connectionId]', 'Connection id to ingest', parsePublicIngestConnectionId)
.option('--all', 'Ingest every eligible configured source', false)
.option('--json', 'Print JSON output', false)
.option('--no-input', 'Disable interactive terminal input')
.addHelpText(
'after',
[
'',
'Examples:',
' klo ingest <connectionId> [options]',
' klo ingest --all [options]',
' klo ingest status [runId] [options]',
' klo ingest watch [runId] [options]',
'',
'Project directory defaults to KLO_PROJECT_DIR when set, otherwise the current working directory.',
'',
].join('\n'),
)
.showHelpAfterError()
.hook('preAction', (_thisCommand, actionCommand) => {
context.writeDebug?.('ingest', actionCommand);
})
.action(async (connectionId: string | undefined, _options: PublicIngestOptions, command) => {
const options = command.opts();
if (options.all === true && connectionId) {
throw new Error('klo ingest accepts either --all or <connectionId>, not both');
}
const args = publicIngestRunCommandSchema.parse({
command: 'run',
projectDir: resolveCommandProjectDir(command),
...(connectionId ? { targetConnectionId: connectionId } : {}),
all: options.all === true,
json: options.json === true,
inputMode: inputMode(options),
});
await runPublicIngestArgs(context, args);
});
ingest
.command('status')
.description('Print status for the latest or selected public ingest run')
.argument('[runId]', 'Public ingest run id')
.option('--json', 'Print JSON output', false)
.option('--no-input', 'Disable interactive terminal input')
.action(async (runId: string | undefined, _options: PublicIngestOptions, command) => {
const options = (command.optsWithGlobals ? command.optsWithGlobals() : command.opts()) as PublicIngestOptions;
const args = publicIngestReadCommandSchema.parse({
command: 'status',
projectDir: resolveCommandProjectDir(command),
...(runId ? { runId } : {}),
json: options.json === true,
inputMode: inputMode(options),
});
await runPublicIngestArgs(context, args);
});
ingest
.command('watch')
.description('Open the latest or selected public ingest visual report')
.argument('[runId]', 'Public ingest run id')
.option('--json', 'Print JSON output instead of the visual report', false)
.option('--no-input', 'Disable interactive terminal input')
.action(async (runId: string | undefined, _options: PublicIngestOptions, command) => {
const options = (command.optsWithGlobals ? command.optsWithGlobals() : command.opts()) as PublicIngestOptions;
const args = publicIngestReadCommandSchema.parse({
command: 'watch',
projectDir: resolveCommandProjectDir(command),
...(runId ? { runId } : {}),
json: options.json === true,
inputMode: inputMode(options),
});
await runPublicIngestArgs(context, args);
});
}

View file

@ -0,0 +1,353 @@
import { type Command, InvalidArgumentError, Option } from '@commander-js/extra-typings';
import { type KloCliCommandContext, parsePositiveIntegerOption, resolveCommandProjectDir } from '../cli-program.js';
import type { KloScanArgs } from '../scan.js';
import { profileMark } from '../startup-profile.js';
profileMark('module:commands/scan-commands');
async function runScanArgs(context: KloCliCommandContext, args: KloScanArgs): Promise<void> {
const runner = context.deps.scan ?? (await import('../scan.js')).runKloScan;
context.setExitCode(await runner(args, context.io));
}
type KloScanModeOption = Extract<KloScanArgs, { command: 'run' }>['mode'];
function parseScanModeOption(value: string): KloScanModeOption {
if (value === 'structural' || value === 'enriched' || value === 'relationships') {
return value;
}
throw new InvalidArgumentError('Allowed choices are structural, enriched, relationships');
}
type KloRelationshipStatusOption = Extract<KloScanArgs, { command: 'relationships' }>['status'];
type KloRelationshipFeedbackDecisionOption = Extract<KloScanArgs, { command: 'relationshipFeedback' }>['decision'];
function parseRelationshipStatusOption(value: string): KloRelationshipStatusOption {
if (value === 'accepted' || value === 'review' || value === 'rejected' || value === 'skipped' || value === 'all') {
return value;
}
throw new InvalidArgumentError('Allowed choices are accepted, review, rejected, skipped, all');
}
function parseRelationshipFeedbackDecisionOption(value: string): KloRelationshipFeedbackDecisionOption {
if (value === 'accepted' || value === 'rejected' || value === 'all') {
return value;
}
throw new InvalidArgumentError('Allowed choices are accepted, rejected, all');
}
function parseNonEmptyOption(value: string): string {
if (value.trim().length === 0) {
throw new InvalidArgumentError('must not be empty');
}
return value;
}
function parseRelationshipCalibrationThreshold(value: string): number {
const parsed = Number(value);
if (Number.isFinite(parsed) && parsed >= 0 && parsed <= 1) {
return parsed;
}
throw new InvalidArgumentError('Allowed range is 0 through 1');
}
function relationshipDecisionArgs(options: {
accept?: string;
reject?: string;
reviewer?: string;
note?: string;
json?: boolean;
}): Pick<
Extract<KloScanArgs, { command: 'relationshipDecision' }>,
'candidateId' | 'decision' | 'reviewer' | 'note' | 'json'
> | null {
const decisionCount = [options.accept !== undefined, options.reject !== undefined].filter(Boolean).length;
if (decisionCount > 1) {
throw new Error('Only one relationship review decision option can be used: --accept and --reject conflict');
}
if (options.accept !== undefined) {
return {
candidateId: options.accept,
decision: 'accepted',
reviewer: options.reviewer ?? 'klo',
note: options.note ?? null,
json: options.json === true,
};
}
if (options.reject !== undefined) {
return {
candidateId: options.reject,
decision: 'rejected',
reviewer: options.reviewer ?? 'klo',
note: options.note ?? null,
json: options.json === true,
};
}
return null;
}
function collectRelationshipCandidateOption(value: string, previous: string[]): string[] {
return [...previous, parseNonEmptyOption(value)];
}
export function registerScanCommands(program: Command, context: KloCliCommandContext): void {
const scan = program
.command('scan')
.description('Run or inspect standalone connection scans')
.argument('[connectionId]', 'KLO connection id to scan')
.option(
'--mode <mode>',
'Scan mode: structural, enriched, relationships (default: structural)',
parseScanModeOption,
)
.option('--dry-run', 'Run without writing scan results', false)
.option('--database-introspection-url <url>', 'Daemon URL for live-database introspection')
.showHelpAfterError()
.addHelpText(
'after',
'\nProject directory defaults to KLO_PROJECT_DIR when set, otherwise the current working directory.\n',
)
.hook('preAction', (_thisCommand, actionCommand) => {
context.writeDebug?.('scan', actionCommand);
})
.action(async (connectionId: string | undefined, options, command) => {
if (!connectionId) {
scan.outputHelp();
context.io.stderr.write('klo dev scan requires <connectionId> or a subcommand\n');
context.setExitCode(1);
return;
}
const mode = options.mode ?? 'structural';
await runScanArgs(context, {
command: 'run',
projectDir: resolveCommandProjectDir(command),
connectionId,
mode,
detectRelationships: mode === 'relationships',
dryRun: options.dryRun === true,
databaseIntrospectionUrl: options.databaseIntrospectionUrl,
});
});
scan
.command('status')
.description('Print status for a local scan run')
.argument('<runId>', 'Local scan run id')
.addHelpText(
'after',
'\n--project-dir is inherited from `klo dev scan` (default: KLO_PROJECT_DIR or current working directory).\n',
)
.action(async (runId: string, _options: unknown, command) => {
await runScanArgs(context, {
command: 'status',
projectDir: resolveCommandProjectDir(command),
runId,
});
});
scan
.command('report')
.description('Print a local scan report')
.argument('<runId>', 'Local scan run id')
.option('--json', 'Print the raw scan report JSON', false)
.addHelpText(
'after',
'\n--project-dir is inherited from `klo dev scan` (default: KLO_PROJECT_DIR or current working directory).\n',
)
.action(async (runId: string, options, command) => {
await runScanArgs(context, {
command: 'report',
projectDir: resolveCommandProjectDir(command),
runId,
json: options.json === true,
});
});
scan
.command('relationships')
.description('Print relationship artifacts for a local scan run')
.argument('<runId>', 'Local scan run id')
.option(
'--status <status>',
'Relationship status: accepted, review, rejected, skipped, all',
parseRelationshipStatusOption,
'review',
)
.option('--limit <count>', 'Maximum relationships to print per status', parsePositiveIntegerOption, 25)
.addOption(
new Option('--accept <candidateId>', 'Record a reviewer accepted decision for a relationship candidate')
.argParser(parseNonEmptyOption)
.conflicts('reject'),
)
.addOption(
new Option('--reject <candidateId>', 'Record a reviewer rejected decision for a relationship candidate')
.argParser(parseNonEmptyOption)
.conflicts('accept'),
)
.option('--note <text>', 'Attach a note when recording a relationship review decision')
.option('--reviewer <name>', 'Reviewer name for a relationship review decision')
.option('--json', 'Print relationship artifacts as JSON', false)
.addHelpText(
'after',
'\n--project-dir is inherited from `klo dev scan` (default: KLO_PROJECT_DIR or current working directory).\n',
)
.action(async (runId: string, options, command) => {
const decision = relationshipDecisionArgs(options);
if (decision) {
await runScanArgs(context, {
command: 'relationshipDecision',
projectDir: resolveCommandProjectDir(command),
runId,
candidateId: decision.candidateId,
decision: decision.decision,
reviewer: decision.reviewer,
note: decision.note,
json: decision.json,
});
return;
}
await runScanArgs(context, {
command: 'relationships',
projectDir: resolveCommandProjectDir(command),
runId,
status: options.status,
json: options.json === true,
limit: options.limit,
});
});
scan
.command('relationship-apply')
.description('Apply accepted relationship review decisions as manual manifest joins')
.argument('<runId>', 'Local scan run id')
.option('--all-accepted', 'Apply all accepted relationship review decisions for the scan run', false)
.option(
'--candidate <candidateId>',
'Apply one accepted relationship review decision',
collectRelationshipCandidateOption,
[],
)
.option('--dry-run', 'Preview relationships that would be written without rewriting manifest shards', false)
.option('--json', 'Print the apply result as JSON', false)
.addHelpText(
'after',
'\n--project-dir is inherited from `klo dev scan` (default: KLO_PROJECT_DIR or current working directory).\n',
)
.action(async (runId: string, options, command) => {
const parentOptions = command.parent?.opts() as { dryRun?: boolean } | undefined;
await runScanArgs(context, {
command: 'relationshipApply',
projectDir: resolveCommandProjectDir(command),
runId,
applyAllAccepted: options.allAccepted === true,
candidateIds: options.candidate,
dryRun: options.dryRun === true || parentOptions?.dryRun === true,
json: options.json === true,
});
});
scan
.command('relationship-feedback')
.description('Export persisted relationship review decisions as calibration labels')
.option('--connection <connectionId>', 'Only export labels for one KLO connection')
.option(
'--decision <decision>',
'Relationship feedback decision: accepted, rejected, all',
parseRelationshipFeedbackDecisionOption,
'all',
)
.addOption(new Option('--json', 'Print the export as JSON').default(false).conflicts('jsonl'))
.addOption(new Option('--jsonl', 'Print labels as newline-delimited JSON').default(false).conflicts('json'))
.addHelpText(
'after',
'\n--project-dir is inherited from `klo dev scan` (default: KLO_PROJECT_DIR or current working directory).\n',
)
.action(async (options, command) => {
await runScanArgs(context, {
command: 'relationshipFeedback',
projectDir: resolveCommandProjectDir(command),
connectionId: options.connection ?? null,
decision: options.decision,
json: options.json === true,
jsonl: options.jsonl === true,
});
});
scan
.command('relationship-calibration')
.description('Summarize relationship feedback labels against current score thresholds')
.option('--connection <connectionId>', 'Only calibrate labels for one KLO connection')
.option(
'--decision <decision>',
'Relationship feedback decision: accepted, rejected, all',
parseRelationshipFeedbackDecisionOption,
'all',
)
.option(
'--accept-threshold <value>',
'Score threshold treated as predicted accepted',
parseRelationshipCalibrationThreshold,
0.85,
)
.option(
'--review-threshold <value>',
'Score threshold treated as predicted review',
parseRelationshipCalibrationThreshold,
0.55,
)
.option('--json', 'Print the calibration report as JSON', false)
.addHelpText(
'after',
'\n--project-dir is inherited from `klo dev scan` (default: KLO_PROJECT_DIR or current working directory).\n',
)
.action(async (options, command) => {
await runScanArgs(context, {
command: 'relationshipCalibration',
projectDir: resolveCommandProjectDir(command),
connectionId: options.connection ?? null,
decision: options.decision,
acceptThreshold: options.acceptThreshold,
reviewThreshold: options.reviewThreshold,
json: options.json === true,
});
});
scan
.command('relationship-thresholds')
.description('Evaluate relationship feedback labels for offline threshold advice')
.option('--connection <connectionId>', 'Only evaluate labels for one KLO connection')
.option(
'--min-total-labels <count>',
'Minimum scored labels before advice can be ready',
parsePositiveIntegerOption,
20,
)
.option(
'--min-accepted-labels <count>',
'Minimum accepted labels before advice can be ready',
parsePositiveIntegerOption,
5,
)
.option(
'--min-rejected-labels <count>',
'Minimum rejected labels before advice can be ready',
parsePositiveIntegerOption,
5,
)
.option('--json', 'Print the threshold advice report as JSON', false)
.addHelpText(
'after',
'\n--project-dir is inherited from `klo dev scan` (default: KLO_PROJECT_DIR or current working directory).\n',
)
.action(async (options, command) => {
await runScanArgs(context, {
command: 'relationshipThresholds',
projectDir: resolveCommandProjectDir(command),
connectionId: options.connection ?? null,
minTotalLabels: options.minTotalLabels,
minAcceptedLabels: options.minAcceptedLabels,
minRejectedLabels: options.minRejectedLabels,
json: options.json === true,
});
});
}

View file

@ -0,0 +1,47 @@
import { type Command, InvalidArgumentError } from '@commander-js/extra-typings';
import { type KloCliCommandContext, resolveCommandProjectDir } from '../cli-program.js';
import type { KloServeArgs } from '../serve.js';
import { profileMark } from '../startup-profile.js';
profileMark('module:commands/serve-commands');
function parseMcp(value: string): 'stdio' {
if (value === 'stdio') {
return 'stdio';
}
throw new InvalidArgumentError('Only stdio is supported in this phase');
}
export function registerServeCommands(program: Command, context: KloCliCommandContext): void {
program
.command('serve')
.description('Run standalone KLO services such as MCP stdio')
.requiredOption('--mcp <mode>', 'MCP transport mode', parseMcp)
.option('--user-id <id>', 'Local user id', 'local')
.option('--semantic-compute', 'Enable semantic-layer compute', false)
.option('--semantic-compute-url <url>', 'HTTP semantic-layer compute URL')
.option('--database-introspection-url <url>', 'Daemon URL for live-database introspection')
.option('--execute-queries', 'Allow semantic-layer query execution', false)
.option('--memory-capture', 'Enable memory capture', false)
.option('--memory-model <model>', 'Memory capture model')
.showHelpAfterError()
.action(async (options, command): Promise<void> => {
const semanticCompute = options.semanticCompute === true || Boolean(options.semanticComputeUrl);
if (options.executeQueries === true && !semanticCompute) {
throw new Error('--execute-queries requires --semantic-compute');
}
const args: KloServeArgs = {
mcp: options.mcp,
projectDir: resolveCommandProjectDir(command),
userId: options.userId,
semanticCompute,
semanticComputeUrl: options.semanticComputeUrl,
databaseIntrospectionUrl: options.databaseIntrospectionUrl,
executeQueries: options.executeQueries === true,
memoryCapture: options.memoryCapture === true,
memoryModel: options.memoryModel,
};
const runner = context.deps.serveStdio ?? (await import('../serve.js')).runKloServeStdio;
context.setExitCode(await runner(args));
});
}

View file

@ -0,0 +1,517 @@
import { type Command, InvalidArgumentError, Option } from '@commander-js/extra-typings';
import type { KloCliCommandContext } from '../cli-program.js';
import { resolveCommandProjectDir } from '../cli-program.js';
import type { KloSetupDatabaseDriver } from '../setup-databases.js';
import type { KloSetupSourceType } from '../setup-sources.js';
import { registerDemoCommands } from './demo-commands.js';
async function runSetupArgs(
context: KloCliCommandContext,
args: Parameters<NonNullable<typeof context.deps.setup>>[0],
) {
const runner = context.deps.setup ?? (await import('../setup.js')).runKloSetup;
context.setExitCode(await runner(args, context.io));
}
function positiveInteger(value: string): number {
const parsed = Number.parseInt(value, 10);
if (!Number.isInteger(parsed) || parsed <= 0) {
throw new Error(`Expected a positive integer, received ${value}`);
}
return parsed;
}
function embeddingBackend(value: string): 'openai' | 'sentence-transformers' {
if (value === 'openai' || value === 'sentence-transformers') {
return value;
}
throw new InvalidArgumentError(`invalid choice '${value}'`);
}
function databaseDriver(value: string): KloSetupDatabaseDriver {
if (
value === 'sqlite' ||
value === 'postgres' ||
value === 'mysql' ||
value === 'clickhouse' ||
value === 'sqlserver' ||
value === 'bigquery' ||
value === 'snowflake'
) {
return value;
}
throw new InvalidArgumentError(`invalid choice '${value}'`);
}
function sourceType(value: string): KloSetupSourceType {
if (
value === 'dbt' ||
value === 'metricflow' ||
value === 'metabase' ||
value === 'looker' ||
value === 'lookml' ||
value === 'notion'
) {
return value;
}
throw new InvalidArgumentError(`invalid choice '${value}'`);
}
function agentScope(value: string): 'project' | 'global' {
if (value === 'project' || value === 'global') {
return value;
}
throw new InvalidArgumentError(`invalid choice '${value}'`);
}
function agentInstallMode(value: string): 'cli' | 'mcp' | 'both' {
if (value === 'cli' || value === 'mcp' || value === 'both') {
return value;
}
throw new InvalidArgumentError(`invalid choice '${value}'`);
}
function positiveNumber(value: string): number {
const parsed = Number.parseInt(value, 10);
if (!Number.isInteger(parsed) || parsed <= 0) {
throw new InvalidArgumentError(`Expected a positive integer, received ${value}`);
}
return parsed;
}
function optionWasSpecified(command: Command, optionName: string): boolean {
const commandWithSources = command as Command & {
getOptionValueSource?: (name: string) => string | undefined;
getOptionValueSourceWithGlobals?: (name: string) => string | undefined;
};
const source =
commandWithSources.getOptionValueSourceWithGlobals?.(optionName) ??
commandWithSources.getOptionValueSource?.(optionName);
return source !== undefined && source !== 'default';
}
function shouldShowSetupEntryMenu(
options: {
new?: boolean;
existing?: boolean;
agents?: boolean;
target?: string;
global?: boolean;
project?: boolean;
skipAgents?: boolean;
yes?: boolean;
input?: boolean;
anthropicApiKeyEnv?: string;
anthropicApiKeyFile?: string;
anthropicModel?: string;
skipLlm?: boolean;
embeddingBackend?: string;
embeddingApiKeyEnv?: string;
embeddingApiKeyFile?: string;
skipEmbeddings?: boolean;
database?: KloSetupDatabaseDriver[];
databaseConnectionId?: string[];
newDatabaseConnectionId?: string;
databaseUrl?: string;
databaseSchema?: string[];
enableHistoricSql?: boolean;
disableHistoricSql?: boolean;
historicSqlWindowDays?: number;
historicSqlMinCalls?: number;
historicSqlServiceAccountPattern?: string[];
historicSqlRedactionPattern?: string[];
skipDatabases?: boolean;
source?: KloSetupSourceType;
sourceConnectionId?: string;
sourcePath?: string;
sourceGitUrl?: string;
sourceBranch?: string;
sourceSubpath?: string;
sourceAuthTokenRef?: string;
sourceUrl?: string;
sourceApiKeyRef?: string;
sourceClientId?: string;
sourceClientSecretRef?: string;
sourceWarehouseConnectionId?: string;
sourceProjectName?: string;
sourceProfilesPath?: string;
sourceTarget?: string;
metabaseDatabaseId?: number;
notionCrawlMode?: string;
notionRootPageId?: string[];
skipInitialSourceIngest?: boolean;
skipSources?: boolean;
},
command: Command,
): boolean {
if (options.database && options.database.length > 0) {
return false;
}
if (options.databaseConnectionId && options.databaseConnectionId.length > 0) {
return false;
}
if (options.databaseSchema && options.databaseSchema.length > 0) {
return false;
}
if (options.historicSqlServiceAccountPattern && options.historicSqlServiceAccountPattern.length > 0) {
return false;
}
if (options.historicSqlRedactionPattern && options.historicSqlRedactionPattern.length > 0) {
return false;
}
if (options.notionRootPageId && options.notionRootPageId.length > 0) {
return false;
}
return ![
'new',
'existing',
'agents',
'target',
'global',
'project',
'skipAgents',
'yes',
'input',
'anthropicApiKeyEnv',
'anthropicApiKeyFile',
'anthropicModel',
'skipLlm',
'embeddingBackend',
'embeddingApiKeyEnv',
'embeddingApiKeyFile',
'skipEmbeddings',
'newDatabaseConnectionId',
'databaseUrl',
'enableHistoricSql',
'disableHistoricSql',
'historicSqlWindowDays',
'historicSqlMinCalls',
'skipDatabases',
'source',
'sourceConnectionId',
'sourcePath',
'sourceGitUrl',
'sourceBranch',
'sourceSubpath',
'sourceAuthTokenRef',
'sourceUrl',
'sourceApiKeyRef',
'sourceClientId',
'sourceClientSecretRef',
'sourceWarehouseConnectionId',
'sourceProjectName',
'sourceProfilesPath',
'sourceTarget',
'metabaseDatabaseId',
'notionCrawlMode',
'skipInitialSourceIngest',
'skipSources',
].some((optionName) => optionWasSpecified(command, optionName));
}
export function registerSetupCommands(program: Command, context: KloCliCommandContext): void {
const setup = program
.command('setup')
.description('Set up or resume a local KLO project')
.option('--project-dir <path>', 'KLO project directory')
.option('--new', 'Create a new KLO project before setup', false)
.option('--existing', 'Use an existing KLO project', false)
.option('--agents', 'Install agent integration only', false)
.addOption(
new Option('--target <target>', 'Agent target').choices([
'claude-code',
'codex',
'cursor',
'opencode',
'universal',
]),
)
.addOption(new Option('--agent-scope <scope>', 'Agent install scope').argParser(agentScope).default('project'))
.option('--project', 'Install agent integration into the project scope', false)
.option('--global', 'Install agent integration into the global target scope', false)
.addOption(
new Option('--agent-install-mode <mode>', 'Agent install mode').argParser(agentInstallMode).default('cli'),
)
.option('--skip-agents', 'Leave agent integration incomplete for now', false)
.option('--yes', 'Accept safe defaults in non-interactive setup', false)
.option('--no-input', 'Disable interactive terminal input')
.option('--anthropic-api-key-env <name>', 'Environment variable containing the Anthropic API key')
.option('--anthropic-api-key-file <path>', 'File containing the Anthropic API key')
.option('--anthropic-model <model>', 'Anthropic model ID to validate and save')
.addOption(new Option('--skip-llm', 'Leave LLM setup incomplete for now').hideHelp().default(false))
.addOption(new Option('--embedding-backend <backend>', 'Embedding backend').argParser(embeddingBackend))
.option('--embedding-api-key-env <name>', 'Environment variable containing the embedding provider API key')
.option('--embedding-api-key-file <path>', 'File containing the embedding provider API key')
.addOption(new Option('--skip-embeddings', 'Leave embedding setup incomplete for now').hideHelp().default(false))
.option(
'--database <driver>',
'Database driver to configure; repeatable',
(value, previous: KloSetupDatabaseDriver[]) => {
return [...previous, databaseDriver(value)];
},
[] as KloSetupDatabaseDriver[],
)
.option(
'--database-connection-id <id>',
'Existing selected connection id or new connection id',
(value, previous: string[]) => [...previous, value],
[],
)
.option('--new-database-connection-id <id>', 'Connection id for one new database connection', (value) => {
if (!/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(value)) {
throw new InvalidArgumentError(`Unsafe connection id: ${value}`);
}
return value;
})
.option('--database-url <url>', 'URL, env:NAME, or file:/path for one new URL-style database connection')
.option(
'--database-schema <schema>',
'Database schema to include; repeatable',
(value, previous: string[]) => [...previous, value],
[],
)
.option('--enable-historic-sql', 'Enable Historic SQL when the selected database supports it', false)
.option('--disable-historic-sql', 'Disable Historic SQL for the selected database', false)
.option('--historic-sql-window-days <number>', 'Historic SQL query-history window', positiveInteger)
.option(
'--historic-sql-min-calls <number>',
'Postgres Historic SQL pg_stat_statements minimum calls floor',
positiveInteger,
)
.option(
'--historic-sql-service-account-pattern <pattern>',
'Historic SQL service-account regex; repeatable',
(value, previous: string[]) => [...previous, value],
[],
)
.option(
'--historic-sql-redaction-pattern <pattern>',
'Historic SQL SQL-literal redaction regex; repeatable',
(value, previous: string[]) => [...previous, value],
[],
)
.option('--skip-databases', 'Leave database setup incomplete; KLO cannot work until a primary source is added', false)
.addOption(new Option('--source <type>', 'Source connector type').argParser(sourceType))
.option('--source-connection-id <id>', 'Connection id for source setup')
.option('--source-path <path>', 'Local source path for dbt, MetricFlow, or LookML')
.option('--source-git-url <url>', 'Git URL for dbt, MetricFlow, or LookML')
.option('--source-branch <branch>', 'Git branch for source setup')
.option('--source-subpath <path>', 'Repo subpath for source setup')
.option('--source-auth-token-ref <ref>', 'env: or file: credential ref for source repo auth')
.option('--source-url <url>', 'Source service URL for Metabase or Looker')
.option('--source-api-key-ref <ref>', 'env: or file: API key ref for Metabase or Notion')
.option('--source-client-id <id>', 'Looker client id')
.option('--source-client-secret-ref <ref>', 'env: or file: Looker client secret ref')
.option('--source-warehouse-connection-id <id>', 'Mapped warehouse connection id')
.option('--source-project-name <name>', 'dbt project name override')
.option('--source-profiles-path <path>', 'dbt profiles path')
.option('--source-target <target>', 'dbt target or source-specific mapping target')
.option('--metabase-database-id <id>', 'Metabase database id to map', positiveNumber)
.addOption(
new Option('--notion-crawl-mode <mode>', 'Notion crawl mode').choices(['all_accessible', 'selected_roots']),
)
.option(
'--notion-root-page-id <id>',
'Notion root page id; repeatable',
(value, previous: string[]) => [...previous, value],
[],
)
.option('--skip-initial-source-ingest', 'Validate source setup without building source context during setup', false)
.option('--skip-sources', 'Mark optional source setup complete with no sources', false)
.showHelpAfterError();
setup.hook('preAction', (_thisCommand, actionCommand) => {
context.writeDebug?.('setup', actionCommand);
});
setup.action(async (options, command) => {
if (options.anthropicApiKeyEnv && options.anthropicApiKeyFile) {
context.io.stderr.write(
'Choose only one Anthropic credential source: --anthropic-api-key-env or --anthropic-api-key-file.\n',
);
context.setExitCode(1);
return;
}
if (options.embeddingApiKeyEnv && options.embeddingApiKeyFile) {
context.io.stderr.write(
'Choose only one embedding credential source: --embedding-api-key-env or --embedding-api-key-file.\n',
);
context.setExitCode(1);
return;
}
if (options.enableHistoricSql && options.disableHistoricSql) {
context.io.stderr.write(
'Choose only one Historic SQL action: --enable-historic-sql or --disable-historic-sql.\n',
);
context.setExitCode(1);
return;
}
if (options.sourcePath && options.sourceGitUrl) {
context.io.stderr.write('Choose only one source location: --source-path or --source-git-url.\n');
context.setExitCode(1);
return;
}
if (options.skipSources && options.source) {
context.io.stderr.write('Choose either --source or --skip-sources.\n');
context.setExitCode(1);
return;
}
const mode = options.new ? 'new' : options.existing ? 'existing' : 'auto';
const resolvedAgentScope = options.global ? 'global' : options.agentScope;
await runSetupArgs(context, {
command: 'run',
projectDir: resolveCommandProjectDir(command),
mode,
agents: options.agents === true,
...(options.target ? { target: options.target } : {}),
agentScope: resolvedAgentScope,
agentInstallMode: options.agentInstallMode,
skipAgents: options.skipAgents === true,
inputMode: options.input === false ? 'disabled' : 'auto',
yes: options.yes === true,
...(options.anthropicApiKeyEnv ? { anthropicApiKeyEnv: options.anthropicApiKeyEnv } : {}),
...(options.anthropicApiKeyFile ? { anthropicApiKeyFile: options.anthropicApiKeyFile } : {}),
...(options.anthropicModel ? { anthropicModel: options.anthropicModel } : {}),
skipLlm: options.skipLlm === true,
...(options.embeddingBackend ? { embeddingBackend: options.embeddingBackend } : {}),
...(options.embeddingApiKeyEnv ? { embeddingApiKeyEnv: options.embeddingApiKeyEnv } : {}),
...(options.embeddingApiKeyFile ? { embeddingApiKeyFile: options.embeddingApiKeyFile } : {}),
skipEmbeddings: options.skipEmbeddings === true,
...(options.database.length > 0 ? { databaseDrivers: options.database } : {}),
...(options.databaseConnectionId.length > 0 ? { databaseConnectionIds: options.databaseConnectionId } : {}),
...(options.newDatabaseConnectionId ? { databaseConnectionId: options.newDatabaseConnectionId } : {}),
...(options.databaseUrl ? { databaseUrl: options.databaseUrl } : {}),
databaseSchemas: options.databaseSchema,
...(options.enableHistoricSql ? { enableHistoricSql: true } : {}),
...(options.disableHistoricSql ? { disableHistoricSql: true } : {}),
...(options.historicSqlWindowDays !== undefined ? { historicSqlWindowDays: options.historicSqlWindowDays } : {}),
...(options.historicSqlMinCalls !== undefined ? { historicSqlMinCalls: options.historicSqlMinCalls } : {}),
...(options.historicSqlServiceAccountPattern.length > 0
? { historicSqlServiceAccountPatterns: options.historicSqlServiceAccountPattern }
: {}),
...(options.historicSqlRedactionPattern.length > 0
? { historicSqlRedactionPatterns: options.historicSqlRedactionPattern }
: {}),
skipDatabases: options.skipDatabases === true,
...(options.source ? { source: options.source } : {}),
...(options.sourceConnectionId ? { sourceConnectionId: options.sourceConnectionId } : {}),
...(options.sourcePath ? { sourcePath: options.sourcePath } : {}),
...(options.sourceGitUrl ? { sourceGitUrl: options.sourceGitUrl } : {}),
...(options.sourceBranch ? { sourceBranch: options.sourceBranch } : {}),
...(options.sourceSubpath ? { sourceSubpath: options.sourceSubpath } : {}),
...(options.sourceAuthTokenRef ? { sourceAuthTokenRef: options.sourceAuthTokenRef } : {}),
...(options.sourceUrl ? { sourceUrl: options.sourceUrl } : {}),
...(options.sourceApiKeyRef ? { sourceApiKeyRef: options.sourceApiKeyRef } : {}),
...(options.sourceClientId ? { sourceClientId: options.sourceClientId } : {}),
...(options.sourceClientSecretRef ? { sourceClientSecretRef: options.sourceClientSecretRef } : {}),
...(options.sourceWarehouseConnectionId
? { sourceWarehouseConnectionId: options.sourceWarehouseConnectionId }
: {}),
...(options.sourceProjectName ? { sourceProjectName: options.sourceProjectName } : {}),
...(options.sourceProfilesPath ? { sourceProfilesPath: options.sourceProfilesPath } : {}),
...(options.sourceTarget ? { sourceTarget: options.sourceTarget } : {}),
...(options.metabaseDatabaseId !== undefined ? { metabaseDatabaseId: options.metabaseDatabaseId } : {}),
...(options.notionCrawlMode ? { notionCrawlMode: options.notionCrawlMode } : {}),
...(options.notionRootPageId.length > 0 ? { notionRootPageIds: options.notionRootPageId } : {}),
runInitialSourceIngest: false,
skipSources: options.skipSources === true,
showEntryMenu: shouldShowSetupEntryMenu(options, command),
});
});
registerDemoCommands(setup, context, { description: 'Run the packaged KLO demo from setup' });
const setupContext = setup.command('context').description('Build, inspect, and recover setup-managed KLO context');
function setupContextInputMode(command: {
optsWithGlobals?: () => unknown;
opts?: () => unknown;
}): 'auto' | 'disabled' {
const options = command.optsWithGlobals?.() as { input?: boolean } | undefined;
return options?.input === false ? 'disabled' : 'auto';
}
setupContext
.command('build')
.description('Build agent-ready KLO context for setup')
.option('--no-input', 'Disable interactive terminal input')
.action(async (options: { input?: boolean }, command) => {
await runSetupArgs(context, {
command: 'context-build',
projectDir: resolveCommandProjectDir(command),
inputMode: options.input === false ? 'disabled' : setupContextInputMode(command),
});
});
setupContext
.command('watch')
.description('Watch a setup-managed context build')
.argument('[runId]', 'Setup context build run id')
.option('--no-input', 'Disable interactive terminal input')
.action(async (runId: string | undefined, options: { input?: boolean }, command) => {
await runSetupArgs(context, {
command: 'context-watch',
projectDir: resolveCommandProjectDir(command),
...(runId ? { runId } : {}),
inputMode: options.input === false ? 'disabled' : setupContextInputMode(command),
});
});
setupContext
.command('status')
.description('Print setup-managed context build status')
.argument('[runId]', 'Setup context build run id')
.option('--json', 'Print JSON output', false)
.action(async (runId: string | undefined, options: { json?: boolean }, command) => {
await runSetupArgs(context, {
command: 'context-status',
projectDir: resolveCommandProjectDir(command),
...(runId ? { runId } : {}),
json: options.json === true,
});
});
setupContext
.command('stop')
.description('Request a pause for a setup-managed context build')
.argument('[runId]', 'Setup context build run id')
.option('--force', 'Request the pause without an interactive confirmation', false)
.action(async (runId: string | undefined, _options: { force?: boolean }, command) => {
await runSetupArgs(context, {
command: 'context-stop',
projectDir: resolveCommandProjectDir(command),
...(runId ? { runId } : {}),
});
});
setup
.command('remove')
.description('Remove setup-managed local integrations')
.option('--agents', 'Remove setup-managed agent integration files', false)
.action(async (options: { agents?: boolean }, command) => {
const parentOptions = command.parent?.opts() as { agents?: boolean } | undefined;
if (options.agents !== true && parentOptions?.agents !== true) {
context.io.stderr.write('Choose what to remove: --agents.\n');
context.setExitCode(1);
return;
}
await runSetupArgs(context, {
command: 'remove-agents',
projectDir: resolveCommandProjectDir(command),
});
});
setup
.command('status')
.description('Show setup readiness for the resolved KLO project')
.option('--json', 'Print JSON output', false)
.action(async (options: { json?: boolean }, command) => {
await runSetupArgs(context, {
command: 'status',
projectDir: resolveCommandProjectDir(command),
json: options.json === true,
});
});
}

View file

@ -0,0 +1,148 @@
import { type Command, InvalidArgumentError, Option } from '@commander-js/extra-typings';
import {
collectOption,
type KloCliCommandContext,
parsePositiveIntegerOption,
resolveCommandProjectDir,
} from '../cli-program.js';
import { slQueryCommandSchema } from '../command-schemas.js';
import type { KloSlArgs } from '../sl.js';
import { profileMark } from '../startup-profile.js';
profileMark('module:commands/sl-commands');
function parseOrderBy(value: string): string | { field: string; direction?: string } {
const [field, direction] = value.split(':');
if (!field) {
throw new InvalidArgumentError('requires a field');
}
if (!direction) {
return field;
}
if (direction !== 'asc' && direction !== 'desc') {
throw new InvalidArgumentError('direction must be asc or desc');
}
return { field, direction };
}
function collectOrderBy(
value: string,
previous: Array<string | { field: string; direction?: string }> = [],
): Array<string | { field: string; direction?: string }> {
return [...previous, parseOrderBy(value)];
}
async function runSlArgs(context: KloCliCommandContext, args: KloSlArgs): Promise<void> {
const runner = context.deps.sl ?? (await import('../sl.js')).runKloSl;
context.setExitCode(await runner(args, context.io));
}
export function registerSlCommands(program: Command, context: KloCliCommandContext, commandName = 'sl'): void {
const sl = program
.command(commandName)
.description('List, read, validate, query, or write local semantic-layer sources')
.showHelpAfterError()
.addHelpText(
'after',
'\nProject directory defaults to KLO_PROJECT_DIR when set, otherwise the current working directory.\n',
);
sl.command('list')
.description('List semantic-layer sources')
.option('--connection-id <id>', 'KLO connection id')
.addOption(
new Option('--output <mode>', 'Output mode: pretty (default in TTY), plain (TSV), or json').choices([
'pretty',
'plain',
'json',
]),
)
.option('--json', 'Shortcut for --output=json (overrides --output)', false)
.action(async (options: { connectionId?: string; output?: 'pretty' | 'plain' | 'json'; json?: boolean }, command) => {
await runSlArgs(context, {
command: 'list',
projectDir: resolveCommandProjectDir(command),
connectionId: options.connectionId,
output: options.output,
json: options.json,
});
});
sl.command('read')
.description('Read a semantic-layer source')
.argument('<sourceName>', 'Semantic-layer source name')
.requiredOption('--connection-id <id>', 'KLO connection id')
.action(async (sourceName: string, options: { connectionId: string }, command) => {
await runSlArgs(context, {
command: 'read',
projectDir: resolveCommandProjectDir(command),
connectionId: options.connectionId,
sourceName,
});
});
sl.command('validate')
.description('Validate a semantic-layer source')
.argument('<sourceName>', 'Semantic-layer source name')
.requiredOption('--connection-id <id>', 'KLO connection id')
.action(async (sourceName: string, options: { connectionId: string }, command) => {
await runSlArgs(context, {
command: 'validate',
projectDir: resolveCommandProjectDir(command),
connectionId: options.connectionId,
sourceName,
});
});
sl.command('write')
.description('Write a semantic-layer source')
.argument('<sourceName>', 'Semantic-layer source name')
.requiredOption('--connection-id <id>', 'KLO connection id')
.requiredOption('--yaml <yaml>', 'Semantic-layer source YAML')
.action(async (sourceName: string, options: { connectionId: string; yaml: string }, command) => {
await runSlArgs(context, {
command: 'write',
projectDir: resolveCommandProjectDir(command),
connectionId: options.connectionId,
sourceName,
yaml: options.yaml,
});
});
sl.command('query')
.description('Compile or execute a semantic-layer query')
.option('--connection-id <id>', 'KLO connection id')
.option('--measure <measure>', 'Measure to query; repeatable', collectOption, [])
.option('--dimension <dimension>', 'Dimension to include; repeatable', collectOption, [])
.option('--filter <filter>', 'Filter expression; repeatable', collectOption, [])
.option('--segment <segment>', 'Segment to include; repeatable', collectOption, [])
.option('--order-by <field[:direction]>', 'Order field, optionally suffixed with :asc or :desc', collectOrderBy, [])
.option('--limit <n>', 'Query limit', parsePositiveIntegerOption)
.option('--include-empty', 'Include empty rows', false)
.addOption(new Option('--format <format>', 'json or sql').choices(['json', 'sql']).default('json'))
.option('--execute', 'Execute the compiled query', false)
.option('--max-rows <n>', 'Maximum rows to return when executing', parsePositiveIntegerOption)
.action(async (options, command) => {
if (options.measure.length === 0) {
throw new Error('sl query requires at least one --measure');
}
const args = slQueryCommandSchema.parse({
command: 'query',
projectDir: resolveCommandProjectDir(command),
connectionId: options.connectionId,
query: {
measures: options.measure,
dimensions: options.dimension,
...(options.filter.length > 0 ? { filters: options.filter } : {}),
...(options.segment.length > 0 ? { segments: options.segment } : {}),
...(options.orderBy.length > 0 ? { order_by: options.orderBy } : {}),
...(options.limit !== undefined ? { limit: options.limit } : {}),
...(options.includeEmpty === true ? { include_empty: true } : {}),
},
format: options.format,
execute: options.execute === true,
...(options.maxRows !== undefined ? { maxRows: options.maxRows } : {}),
});
await runSlArgs(context, args);
});
}

View file

@ -0,0 +1,23 @@
import type { Command } from '@commander-js/extra-typings';
import type { KloCliCommandContext } from '../cli-program.js';
import { resolveCommandProjectDir } from '../cli-program.js';
export function registerStatusCommands(program: Command, context: KloCliCommandContext): void {
program
.command('status')
.description('Show current KLO project setup status')
.option('--json', 'Print JSON output', false)
.action(async (options: { json?: boolean }, command) => {
const runner = context.deps.setup ?? (await import('../setup.js')).runKloSetup;
context.setExitCode(
await runner(
{
command: 'status',
projectDir: resolveCommandProjectDir(command),
json: options.json === true,
},
context.io,
),
);
});
}

View file

@ -0,0 +1,353 @@
import { mkdir, readFile, writeFile } from 'node:fs/promises';
import { homedir } from 'node:os';
import { dirname, join } from 'node:path';
import type { CommandUnknownOpts, Option } from '@commander-js/extra-typings';
export interface CompletionRequest {
position: number;
words: string[];
}
interface CompletionCandidate {
value: string;
description?: string;
}
interface CommandWithHiddenFlag extends CommandUnknownOpts {
_hidden?: boolean;
}
interface ResolveState {
command: CommandUnknownOpts;
pendingOption?: Option;
positionalIndex: number;
}
export interface ZshCompletionInstallResult {
completionPath: string;
zshrcPath: string;
}
const KLO_COMPLETION_BLOCK_START = '# >>> klo completion >>>';
const KLO_COMPLETION_BLOCK_END = '# <<< klo completion <<<';
const KLO_COMPLETION_BLOCK_PATTERN = new RegExp(
`\\n?${escapeRegExp(KLO_COMPLETION_BLOCK_START)}[\\s\\S]*?${escapeRegExp(KLO_COMPLETION_BLOCK_END)}\\n?`,
'g',
);
export function zshCompletionScript(): string {
const zshWords = '$' + '{words[@]}';
const zshCompletionCapture = [
'$',
`{(@f)$("${'$'}{klo_completion_command[@]}" dev __complete --shell zsh --position "$CURRENT" -- "${zshWords}" 2>/dev/null)}`,
].join('');
const zshCompletionsCount = '$' + '{#completions[@]}';
const zshCompletionCommand = '$' + '(eval "print -r -- $' + '{KLO_COMPLETION_COMMAND:-klo}")';
return [
'#compdef klo',
'',
'_klo() {',
' local -a completions',
' local -a klo_completion_command',
` klo_completion_command=("\${(@z)${zshCompletionCommand}}")`,
` completions=("${zshCompletionCapture}")`,
` if (( ${zshCompletionsCount} )); then`,
" _describe 'klo completions' completions",
' else',
' _files',
' fi',
'}',
'',
'compdef _klo klo',
'',
].join('\n');
}
export async function installZshCompletion(): Promise<ZshCompletionInstallResult> {
const homeDir = process.env.HOME || homedir();
const zshConfigDir = process.env.ZDOTDIR || homeDir;
const completionDir = join(homeDir, '.zfunc');
const completionPath = join(completionDir, '_klo');
const zshrcPath = join(zshConfigDir, '.zshrc');
await mkdir(completionDir, { recursive: true });
await mkdir(dirname(zshrcPath), { recursive: true });
await writeFile(completionPath, zshCompletionScript(), 'utf-8');
const existingZshrc = await readOptionalTextFile(zshrcPath);
const nextZshrc = updateZshrcCompletionBlock(existingZshrc);
await writeFile(zshrcPath, nextZshrc, 'utf-8');
return { completionPath, zshrcPath };
}
export function completeCommanderInput(program: CommandUnknownOpts, request: CompletionRequest): string[] {
const words = completionWordsForPosition(request.words, request.position);
const tokens = stripProgramName(program, words);
const current = tokens.at(-1) ?? '';
const previous = tokens.slice(0, -1);
const state = resolveCommandState(program, previous);
return candidatesForState(state, current).map(formatZshCandidate);
}
function completionWordsForPosition(words: string[], position: number): string[] {
if (!Number.isInteger(position) || position < 1) {
return words;
}
return words.slice(0, position);
}
function stripProgramName(program: CommandUnknownOpts, words: string[]): string[] {
const [first, ...rest] = words;
if (!first) {
return [];
}
return first === program.name() || first.endsWith(`/${program.name()}`) ? rest : words;
}
function resolveCommandState(program: CommandUnknownOpts, tokens: string[]): ResolveState {
let command = program;
let positionalIndex = 0;
let pendingOption: Option | undefined;
let positionalOnly = false;
for (let index = 0; index < tokens.length; index += 1) {
const token = tokens[index];
if (pendingOption) {
pendingOption = undefined;
continue;
}
if (token === '--') {
positionalOnly = true;
continue;
}
if (!positionalOnly && token.startsWith('-')) {
const option = findOption(command, optionNameFromToken(token));
if (option && !token.includes('=') && optionTakesValue(option)) {
if (index === tokens.length - 1) {
pendingOption = option;
} else if (option.required || !tokens[index + 1]?.startsWith('-')) {
index += 1;
}
}
continue;
}
const child = findVisibleSubcommand(command, token);
if (child) {
command = child;
positionalIndex = 0;
continue;
}
positionalIndex += 1;
}
return { command, pendingOption, positionalIndex };
}
function candidatesForState(state: ResolveState, current: string): CompletionCandidate[] {
const optionValue = splitOptionValueToken(current);
if (optionValue) {
const option = findOption(state.command, optionValue.optionName);
return choiceCandidates(option?.argChoices, optionValue.valuePrefix, optionValue.optionPrefix);
}
if (state.pendingOption) {
return choiceCandidates(state.pendingOption.argChoices, current);
}
if (current.startsWith('-')) {
return visibleOptions(state.command)
.map(optionCandidate)
.filter((candidate) => candidate.value.startsWith(current));
}
const commandCandidates = visibleSubcommands(state.command)
.map(commandCandidate)
.filter((candidate) => candidate.value.startsWith(current));
const argument = state.command.registeredArguments[state.positionalIndex];
return [...commandCandidates, ...choiceCandidates(argument?.argChoices, current)];
}
function visibleSubcommands(command: CommandUnknownOpts): CommandUnknownOpts[] {
return command.commands.filter((subcommand) => (subcommand as CommandWithHiddenFlag)._hidden !== true);
}
function findVisibleSubcommand(command: CommandUnknownOpts, name: string): CommandUnknownOpts | undefined {
return visibleSubcommands(command).find(
(subcommand) => subcommand.name() === name || subcommand.aliases().includes(name),
);
}
function visibleOptions(command: CommandUnknownOpts): Option[] {
const options: Option[] = [];
const seen = new Set<string>();
for (const current of commandChain(command)) {
for (const option of current.options) {
if (option.hidden) {
continue;
}
const key = option.long ?? option.short ?? option.flags;
if (seen.has(key)) {
continue;
}
seen.add(key);
options.push(option);
}
}
return options;
}
function commandChain(command: CommandUnknownOpts): CommandUnknownOpts[] {
const chain: CommandUnknownOpts[] = [];
let current: CommandUnknownOpts | null = command;
while (current) {
chain.unshift(current);
current = current.parent;
}
return chain;
}
function findOption(command: CommandUnknownOpts, name: string): Option | undefined {
return visibleOptions(command).find((option) => option.long === name || option.short === name);
}
function optionTakesValue(option: Option): boolean {
return option.required || option.optional;
}
function optionNameFromToken(token: string): string {
return token.split('=', 1)[0] ?? token;
}
function splitOptionValueToken(
token: string,
): { optionName: string; optionPrefix: string; valuePrefix: string } | null {
const separatorIndex = token.indexOf('=');
if (!token.startsWith('-') || separatorIndex < 0) {
return null;
}
return {
optionName: token.slice(0, separatorIndex),
optionPrefix: token.slice(0, separatorIndex + 1),
valuePrefix: token.slice(separatorIndex + 1),
};
}
function commandCandidate(command: CommandUnknownOpts): CompletionCandidate {
return {
value: command.name(),
description: command.summary() || command.description(),
};
}
function optionCandidate(option: Option): CompletionCandidate {
return {
value: option.long ?? option.short ?? option.flags,
description: option.description,
};
}
function choiceCandidates(
choices: readonly string[] | undefined,
prefix: string,
completionPrefix = '',
): CompletionCandidate[] {
return (choices ?? [])
.filter((choice) => choice.startsWith(prefix))
.map((choice) => ({ value: `${completionPrefix}${choice}` }));
}
function formatZshCandidate(candidate: CompletionCandidate): string {
if (!candidate.description) {
return escapeZshCompletion(candidate.value);
}
return `${escapeZshCompletion(candidate.value)}:${escapeZshDescription(candidate.description)}`;
}
function escapeZshCompletion(value: string): string {
return value.replace(/\\/g, '\\\\').replace(/:/g, '\\:');
}
function escapeZshDescription(value: string): string {
return value.replace(/\s+/g, ' ').replace(/\\/g, '\\\\').replace(/:/g, '\\:').trim();
}
async function readOptionalTextFile(path: string): Promise<string> {
try {
return await readFile(path, 'utf-8');
} catch (error) {
if (isNodeError(error) && error.code === 'ENOENT') {
return '';
}
throw error;
}
}
function updateZshrcCompletionBlock(contents: string): string {
const withoutManagedBlock = contents.replace(KLO_COMPLETION_BLOCK_PATTERN, normalizeTrailingNewline);
const hasCompinit = /^.*\bcompinit\b.*$/m.test(withoutManagedBlock);
const block = zshrcCompletionBlock({ includeCompinit: !hasCompinit });
if (!hasCompinit) {
return appendBlock(withoutManagedBlock, block);
}
const compinitMatch = /^.*\bcompinit\b.*$/m.exec(withoutManagedBlock);
if (!compinitMatch || compinitMatch.index === undefined) {
return appendBlock(withoutManagedBlock, block);
}
return [
withoutManagedBlock.slice(0, compinitMatch.index),
block,
'\n',
withoutManagedBlock.slice(compinitMatch.index),
].join('');
}
function zshrcCompletionBlock(options: { includeCompinit: boolean }): string {
return [
KLO_COMPLETION_BLOCK_START,
'_klo_completion_command() {',
' local dir="$PWD"',
' while [[ "$dir" != "/" ]]; do',
` if [[ -f "$dir/package.json" ]] && command grep -q '"name": "klo-workspace"' "$dir/package.json" 2>/dev/null; then`,
' print -r -- "node $dir/scripts/run-klo.mjs --"',
' return',
' fi',
' dir="' + '$' + '{dir:h}"',
' done',
' print -r -- "klo"',
'}',
"export KLO_COMPLETION_COMMAND='$(_klo_completion_command)'",
'setopt complete_aliases',
'fpath=("$HOME/.zfunc" $fpath)',
...(options.includeCompinit ? ['autoload -Uz compinit', 'compinit'] : []),
KLO_COMPLETION_BLOCK_END,
].join('\n');
}
function appendBlock(contents: string, block: string): string {
if (!contents.trim()) {
return `${block}\n`;
}
return `${contents.replace(/\s*$/, '\n\n')}${block}\n`;
}
function normalizeTrailingNewline(match: string): string {
return match.startsWith('\n') || match.endsWith('\n') ? '\n' : '';
}
function escapeRegExp(value: string): string {
return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
}
function isNodeError(error: unknown): error is NodeJS.ErrnoException {
return error instanceof Error && 'code' in error;
}

View file

@ -0,0 +1,649 @@
import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { initKloProject, parseKloProjectConfig } from '@klo/context/project';
import type { KloConnectionDriver, KloScanConnector, KloSchemaSnapshot } from '@klo/context/scan';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { runKloConnection } from './connection.js';
import { runKloCli, type KloCliIo } from './index.js';
function makeIo(options: { stdoutIsTty?: boolean; stdinIsTty?: boolean } = {}) {
let stdout = '';
let stderr = '';
return {
io: {
stdin: {
isTTY: options.stdinIsTty,
},
stdout: {
isTTY: options.stdoutIsTty,
write: (chunk: string) => {
stdout += chunk;
},
},
stderr: {
write: (chunk: string) => {
stderr += chunk;
},
},
},
stdout: () => stdout,
stderr: () => stderr,
};
}
function snapshotFor(driver: KloConnectionDriver, tableNames: string[]): KloSchemaSnapshot {
return {
connectionId: 'warehouse',
driver,
extractedAt: '2026-04-29T00:00:00.000Z',
scope: {},
metadata: {},
tables: tableNames.map((name) => ({
catalog: null,
db: null,
name,
kind: 'table',
comment: null,
estimatedRows: null,
columns: [],
foreignKeys: [],
})),
};
}
function nativeConnector(driver: KloConnectionDriver, tableNames: string[]) {
const introspect = vi.fn(async () => snapshotFor(driver, tableNames));
const cleanup = vi.fn(async () => undefined);
const connector: KloScanConnector = {
id: `${driver}:warehouse`,
driver,
capabilities: {
structuralIntrospection: true,
tableSampling: false,
columnSampling: false,
columnStats: false,
readOnlySql: false,
nestedAnalysis: false,
eventStreamDiscovery: false,
formalForeignKeys: false,
estimatedRowCounts: false,
},
introspect,
cleanup,
};
return { connector, introspect, cleanup };
}
describe('runKloConnection', () => {
let tempDir: string;
beforeEach(async () => {
tempDir = await mkdtemp(join(tmpdir(), 'klo-cli-connection-'));
});
afterEach(async () => {
await rm(tempDir, { recursive: true, force: true });
});
it('adds and lists env-referenced connections without resolving secrets', async () => {
const projectDir = join(tempDir, 'project');
await initKloProject({ projectDir, projectName: 'warehouse' });
const io = makeIo();
await expect(
runKloConnection(
{
command: 'add',
projectDir,
driver: 'postgres',
connectionId: 'warehouse',
url: 'env:DATABASE_URL',
schemas: ['public'],
readonly: true,
force: false,
allowLiteralCredentials: false,
},
io.io,
),
).resolves.toBe(0);
expect(io.stdout()).toContain('Connection: warehouse');
await expect(readFile(join(projectDir, 'klo.yaml'), 'utf-8')).resolves.toContain('url: env:DATABASE_URL');
const listIo = makeIo();
await expect(runKloConnection({ command: 'list', projectDir }, listIo.io)).resolves.toBe(0);
expect(listIo.stdout()).toContain('warehouse');
expect(listIo.stdout()).toContain('postgres');
});
it('removes a configured connection from klo.yaml without deleting local artifacts when forced', async () => {
const projectDir = join(tempDir, 'project');
await initKloProject({ projectDir, projectName: 'warehouse' });
await runKloConnection(
{
command: 'add',
projectDir,
driver: 'sqlite',
connectionId: 'warehouse',
url: undefined,
schemas: [],
readonly: true,
force: false,
allowLiteralCredentials: false,
},
makeIo().io,
);
const artifactPath = join(projectDir, '.klo', 'artifacts', 'warehouse.txt');
await mkdir(join(projectDir, '.klo', 'artifacts'), { recursive: true });
await writeFile(artifactPath, 'keep me', 'utf-8');
const io = makeIo();
await expect(
runKloConnection(
{
command: 'remove',
projectDir,
connectionId: 'warehouse',
force: true,
inputMode: 'disabled',
},
io.io,
),
).resolves.toBe(0);
const parsed = parseKloProjectConfig(await readFile(join(projectDir, 'klo.yaml'), 'utf-8'));
expect(parsed.connections.warehouse).toBeUndefined();
await expect(readFile(artifactPath, 'utf-8')).resolves.toBe('keep me');
expect(io.stdout()).toContain('Connection removed from klo.yaml.');
expect(io.stdout()).toContain(
'Ingested artifacts from this connection remain in .klo/. Run klo dev artifacts to inspect.',
);
expect(io.stderr()).toBe('');
});
it('requires --force when removing in non-interactive mode', async () => {
const projectDir = join(tempDir, 'project');
await initKloProject({ projectDir, projectName: 'warehouse' });
await runKloConnection(
{
command: 'add',
projectDir,
driver: 'sqlite',
connectionId: 'warehouse',
url: undefined,
schemas: [],
readonly: true,
force: false,
allowLiteralCredentials: false,
},
makeIo().io,
);
const io = makeIo();
await expect(
runKloConnection(
{
command: 'remove',
projectDir,
connectionId: 'warehouse',
force: false,
inputMode: 'disabled',
},
io.io,
),
).resolves.toBe(1);
expect(io.stderr()).toContain('connection remove warehouse requires --force when input is disabled or not interactive');
});
it('returns a clear error when removing an unknown connection', async () => {
const projectDir = join(tempDir, 'project');
await initKloProject({ projectDir, projectName: 'warehouse' });
const io = makeIo();
await expect(
runKloConnection(
{
command: 'remove',
projectDir,
connectionId: 'missing',
force: true,
inputMode: 'disabled',
},
io.io,
),
).resolves.toBe(1);
expect(io.stderr()).toContain('Connection "missing" is not configured in klo.yaml');
});
it('asks for confirmation before removing in an interactive terminal', async () => {
const projectDir = join(tempDir, 'project');
await initKloProject({ projectDir, projectName: 'warehouse' });
await runKloConnection(
{
command: 'add',
projectDir,
driver: 'sqlite',
connectionId: 'warehouse',
url: undefined,
schemas: [],
readonly: true,
force: false,
allowLiteralCredentials: false,
},
makeIo().io,
);
const io = makeIo({ stdoutIsTty: true, stdinIsTty: true });
const prompts = {
confirm: vi.fn(async () => true),
cancel: vi.fn(),
};
await expect(
runKloConnection(
{
command: 'remove',
projectDir,
connectionId: 'warehouse',
force: false,
},
io.io,
{ prompts },
),
).resolves.toBe(0);
expect(prompts.confirm).toHaveBeenCalledWith({
message: 'Remove connection "warehouse" from klo.yaml? Ingested artifacts will remain in .klo/.',
initialValue: false,
});
});
it('runs public connect map as refresh, validate, and list over the low-level mapping runner', async () => {
const io = makeIo();
const runMapping = vi.fn(async (argv: string[], mappingIo: KloCliIo) => {
if (argv[0] === 'refresh') {
mappingIo.stdout.write('Discovery: 1 database\n');
mappingIo.stdout.write('Unmapped discovered: 1\n');
mappingIo.stdout.write('Stale mappings: 0\n');
return 0;
}
if (argv[0] === 'validate') {
mappingIo.stdout.write('Mapping validation passed: prod-metabase\n');
return 0;
}
if (argv[0] === 'list') {
mappingIo.stdout.write('1 -> [unmapped] (Analytics, sync: on, source: refresh)\n');
return 0;
}
return 1;
});
await expect(
runKloConnection(
{ command: 'map', projectDir: '/tmp/project', sourceConnectionId: 'prod-metabase', json: false },
io.io,
{ runMapping },
),
).resolves.toBe(0);
expect(runMapping).toHaveBeenNthCalledWith(
1,
['refresh', 'prod-metabase', '--auto-accept', '--project-dir', '/tmp/project'],
expect.any(Object),
);
expect(runMapping).toHaveBeenNthCalledWith(
2,
['validate', 'prod-metabase', '--project-dir', '/tmp/project'],
expect.any(Object),
);
expect(runMapping).toHaveBeenNthCalledWith(
3,
['list', 'prod-metabase', '--project-dir', '/tmp/project'],
expect.any(Object),
);
expect(io.stdout()).toContain('Mapping: prod-metabase');
expect(io.stdout()).toContain('Discovery: 1 database');
expect(io.stdout()).toContain('Mappings:');
expect(io.stdout()).toContain('1 -> [unmapped]');
expect(io.stdout()).toContain('Next:');
expect(io.stdout()).toContain('klo ingest prod-metabase');
expect(io.stdout()).toContain('klo dev mapping');
expect(io.stderr()).toBe('');
});
it('prints stable JSON for public connect map without leaking low-level stdout', async () => {
const io = makeIo();
const runMapping = vi.fn(async (argv: string[], mappingIo: KloCliIo) => {
if (argv[0] === 'refresh') {
mappingIo.stdout.write('Discovery: 1 connection\nUnmapped discovered: 0\nStale mappings: 0\n');
return 0;
}
if (argv[0] === 'validate') {
mappingIo.stdout.write('Mapping validation passed: prod-looker\n');
return 0;
}
if (argv[0] === 'list') {
expect(argv).toContain('--json');
mappingIo.stdout.write(
`${JSON.stringify(
[
{
lookerConnectionName: 'analytics',
kloConnectionId: 'prod-warehouse',
source: 'klo.yaml',
},
],
null,
2,
)}\n`,
);
return 0;
}
return 1;
});
await expect(
runKloConnection(
{ command: 'map', projectDir: '/tmp/project', sourceConnectionId: 'prod-looker', json: true },
io.io,
{ runMapping },
),
).resolves.toBe(0);
const parsed = JSON.parse(io.stdout()) as {
connectionId: string;
refresh: { ok: boolean; output: string[] };
validation: { ok: boolean; output: string[] };
mappings: Array<{ lookerConnectionName: string; kloConnectionId: string; source: string }>;
};
expect(parsed).toEqual({
connectionId: 'prod-looker',
refresh: {
ok: true,
output: ['Discovery: 1 connection', 'Unmapped discovered: 0', 'Stale mappings: 0'],
},
validation: {
ok: true,
output: ['Mapping validation passed: prod-looker'],
},
mappings: [
{
lookerConnectionName: 'analytics',
kloConnectionId: 'prod-warehouse',
source: 'klo.yaml',
},
],
});
expect(io.stderr()).toBe('');
});
it('returns the refresh failure when public connect map cannot discover source metadata', async () => {
const io = makeIo();
const runMapping = vi.fn(async (argv: string[], mappingIo: KloCliIo) => {
if (argv[0] === 'refresh') {
mappingIo.stderr.write('Metabase API key is not configured\n');
return 1;
}
return 0;
});
await expect(
runKloConnection(
{ command: 'map', projectDir: '/tmp/project', sourceConnectionId: 'prod-metabase', json: false },
io.io,
{ runMapping },
),
).resolves.toBe(1);
expect(runMapping).toHaveBeenCalledTimes(1);
expect(io.stdout()).toBe('');
expect(io.stderr()).toContain('Metabase API key is not configured');
});
it('rejects literal credential URLs unless explicitly allowed', async () => {
const projectDir = join(tempDir, 'project');
await initKloProject({ projectDir, projectName: 'warehouse' });
const io = makeIo();
await expect(
runKloConnection(
{
command: 'add',
projectDir,
driver: 'postgres',
connectionId: 'warehouse',
url: 'postgres://localhost:5432/warehouse',
schemas: [],
readonly: true,
force: false,
allowLiteralCredentials: false,
},
io.io,
),
).resolves.toBe(1);
expect(io.stderr()).toContain('Literal credential URLs require --allow-literal-credentials');
});
it('warns before writing explicitly allowed literal credential URLs without echoing the URL', async () => {
const projectDir = join(tempDir, 'project');
await initKloProject({ projectDir, projectName: 'warehouse' });
const io = makeIo();
const literalUrl = 'postgres://localhost:5432/warehouse';
await expect(
runKloConnection(
{
command: 'add',
projectDir,
driver: 'postgres',
connectionId: 'warehouse',
url: literalUrl,
schemas: ['public'],
readonly: true,
force: false,
allowLiteralCredentials: true,
},
io.io,
),
).resolves.toBe(0);
expect(io.stderr()).toContain(
'Warning: writing a literal credential URL to klo.yaml for connection "warehouse". Prefer env:NAME or file:/path references.',
);
expect(io.stderr()).not.toContain(literalUrl);
await expect(readFile(join(projectDir, 'klo.yaml'), 'utf-8')).resolves.toContain(literalUrl);
});
it('adds a Notion connection without writing token values', async () => {
const projectDir = join(tempDir, 'project');
await initKloProject({ projectDir, projectName: 'warehouse' });
const io = makeIo();
await expect(
runKloConnection(
{
command: 'add',
projectDir,
driver: 'notion',
connectionId: 'notion-main',
url: undefined,
schemas: [],
readonly: false,
force: false,
allowLiteralCredentials: false,
notion: {
authTokenRef: 'env:NOTION_AUTH_TOKEN',
crawlMode: 'all_accessible',
rootPageIds: [],
rootDatabaseIds: [],
rootDataSourceIds: [],
maxPagesPerRun: 50,
maxKnowledgeCreatesPerRun: 4,
maxKnowledgeUpdatesPerRun: 12,
},
},
io.io,
),
).resolves.toBe(0);
const yaml = await readFile(join(projectDir, 'klo.yaml'), 'utf-8');
expect(yaml).toContain('driver: notion');
expect(yaml).toContain('auth_token_ref: env:NOTION_AUTH_TOKEN');
expect(yaml).toContain('crawl_mode: all_accessible');
expect(yaml).toContain('max_pages_per_run: 50');
expect(yaml).not.toContain('ntn_');
expect(io.stdout()).toContain('Connection: notion-main');
expect(io.stdout()).toContain('Driver: notion');
});
it('runs connection notion pick --no-input through the public connection entrypoint', async () => {
const projectDir = join(tempDir, 'project');
await initKloProject({ projectDir, projectName: 'warehouse' });
await runKloConnection(
{
command: 'add',
projectDir,
driver: 'notion',
connectionId: 'notion-main',
url: undefined,
schemas: [],
readonly: false,
force: false,
allowLiteralCredentials: false,
notion: {
authTokenRef: 'env:NOTION_AUTH_TOKEN',
crawlMode: 'all_accessible',
rootPageIds: [],
rootDatabaseIds: ['database-1'],
rootDataSourceIds: ['data-source-1'],
maxPagesPerRun: 50,
maxKnowledgeCreatesPerRun: 4,
maxKnowledgeUpdatesPerRun: 12,
},
},
makeIo().io,
);
const io = makeIo();
await expect(
runKloCli(
[
'connection',
'notion',
'pick',
'notion-main',
'--project-dir',
projectDir,
'--no-input',
'--root-page-id',
'11111111222233334444555555555555',
],
io.io,
),
).resolves.toBe(0);
const yaml = await readFile(join(projectDir, 'klo.yaml'), 'utf-8');
expect(yaml).toContain('crawl_mode: selected_roots');
expect(yaml).toContain('11111111-2222-3333-4444-555555555555');
expect(yaml).toContain('database-1');
expect(yaml).toContain('data-source-1');
expect(io.stdout()).toContain('Connection: notion-main');
});
it('tests a configured connection through the native scan connector', async () => {
const projectDir = join(tempDir, 'project');
await initKloProject({ projectDir, projectName: 'warehouse' });
await runKloConnection(
{
command: 'add',
projectDir,
driver: 'sqlite',
connectionId: 'warehouse',
url: undefined,
schemas: [],
readonly: true,
force: false,
allowLiteralCredentials: false,
},
makeIo().io,
);
const { connector, introspect, cleanup } = nativeConnector('sqlite', ['customers', 'orders']);
const createScanConnector = vi.fn(async () => connector);
const io = makeIo();
await expect(
runKloConnection({ command: 'test', projectDir, connectionId: 'warehouse' }, io.io, {
createScanConnector,
}),
).resolves.toBe(0);
expect(createScanConnector).toHaveBeenCalledWith(expect.objectContaining({ projectDir }), 'warehouse');
expect(introspect).toHaveBeenCalledWith(
{
connectionId: 'warehouse',
driver: 'sqlite',
mode: 'structural',
dryRun: true,
detectRelationships: false,
},
{ runId: 'connection-test-warehouse' },
);
expect(cleanup).toHaveBeenCalledTimes(1);
expect(io.stdout()).toContain('Connection test passed: warehouse');
expect(io.stdout()).toContain('Driver: sqlite');
expect(io.stdout()).toContain('Tables: 2');
});
it('cleans up the native scan connector when connection testing fails', async () => {
const projectDir = join(tempDir, 'project');
await initKloProject({ projectDir, projectName: 'warehouse' });
await runKloConnection(
{
command: 'add',
projectDir,
driver: 'sqlite',
connectionId: 'warehouse',
url: undefined,
schemas: [],
readonly: true,
force: false,
allowLiteralCredentials: false,
},
makeIo().io,
);
const cleanup = vi.fn(async () => undefined);
const connector: KloScanConnector = {
id: 'sqlite:warehouse',
driver: 'sqlite',
capabilities: {
structuralIntrospection: true,
tableSampling: false,
columnSampling: false,
columnStats: false,
readOnlySql: false,
nestedAnalysis: false,
eventStreamDiscovery: false,
formalForeignKeys: false,
estimatedRowCounts: false,
},
introspect: vi.fn(async () => {
throw new Error('database file is unreadable');
}),
cleanup,
};
const io = makeIo();
await expect(
runKloConnection({ command: 'test', projectDir, connectionId: 'warehouse' }, io.io, {
createScanConnector: vi.fn(async () => connector),
}),
).resolves.toBe(1);
expect(cleanup).toHaveBeenCalledTimes(1);
expect(io.stderr()).toContain('database file is unreadable');
});
});

View file

@ -0,0 +1,415 @@
import { cancel, confirm, isCancel } from '@clack/prompts';
import { type KloLocalProject, loadKloProject, serializeKloProjectConfig } from '@klo/context/project';
import type { KloScanConnector } from '@klo/context/scan';
import type { KloConnectionMappingArgs } from './commands/connection-mapping.js';
import type { KloCliIo } from './index.js';
import { createKloCliScanConnector } from './local-scan-connectors.js';
import { profileMark } from './startup-profile.js';
profileMark('module:connection');
interface KloNotionConnectionCliConfig {
authTokenRef: string;
crawlMode: 'all_accessible' | 'selected_roots';
rootPageIds: string[];
rootDatabaseIds: string[];
rootDataSourceIds: string[];
maxPagesPerRun?: number;
maxKnowledgeCreatesPerRun?: number;
maxKnowledgeUpdatesPerRun?: number;
}
type KloConnectionInputMode = 'disabled';
export type KloConnectionArgs =
| { command: 'list'; projectDir: string }
| {
command: 'add';
projectDir: string;
driver: string;
connectionId: string;
url?: string;
schemas: string[];
readonly: boolean;
force: boolean;
allowLiteralCredentials: boolean;
notion?: KloNotionConnectionCliConfig;
}
| { command: 'test'; projectDir: string; connectionId: string }
| {
command: 'remove';
projectDir: string;
connectionId: string;
force: boolean;
inputMode?: KloConnectionInputMode;
}
| {
command: 'map';
projectDir: string;
sourceConnectionId: string;
json: boolean;
};
interface KloConnectionPromptAdapter {
confirm(options: { message: string; initialValue?: boolean }): Promise<boolean>;
cancel(message: string): void;
}
interface KloConnectionIo extends KloCliIo {
stdin?: { isTTY?: boolean };
}
interface KloConnectionDeps {
createScanConnector?: typeof createKloCliScanConnector;
runMapping?: (argv: string[], io: KloCliIo) => Promise<number>;
prompts?: KloConnectionPromptAdapter;
}
function assertSafeConnectionId(connectionId: string): void {
if (!/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(connectionId)) {
throw new Error(`Unsafe connection id: ${connectionId}`);
}
}
function isCredentialReference(value: string): boolean {
return value.startsWith('env:') || value.startsWith('file:');
}
function literalCredentialWarning(connectionId: string): string {
return `Warning: writing a literal credential URL to klo.yaml for connection "${connectionId}". Prefer env:NAME or file:/path references.`;
}
function createClackConnectionPromptAdapter(): KloConnectionPromptAdapter {
return {
async confirm(options: { message: string; initialValue?: boolean }): Promise<boolean> {
const value = await confirm(options);
return isCancel(value) ? false : value;
},
cancel(message: string): void {
cancel(message);
},
};
}
function isInteractiveConnectionIo(
args: Extract<KloConnectionArgs, { command: 'remove' }>,
io: KloConnectionIo,
): boolean {
return args.inputMode !== 'disabled' && io.stdin?.isTTY === true && io.stdout.isTTY === true;
}
async function cleanupConnector(connector: KloScanConnector | null): Promise<void> {
if (connector?.cleanup) {
await connector.cleanup();
}
}
async function testNativeConnection(
project: KloLocalProject,
connectionId: string,
createScanConnector: typeof createKloCliScanConnector,
): Promise<{ driver: string; tableCount: number }> {
let connector: KloScanConnector | null = null;
try {
connector = await createScanConnector(project, connectionId);
const snapshot = await connector.introspect(
{
connectionId,
driver: connector.driver,
mode: 'structural',
dryRun: true,
detectRelationships: false,
},
{ runId: `connection-test-${connectionId}` },
);
return {
driver: connector.driver,
tableCount: snapshot.tables.length,
};
} finally {
await cleanupConnector(connector);
}
}
interface BufferedIo extends KloCliIo {
stdoutText(): string;
stderrText(): string;
}
function createBufferedIo(): BufferedIo {
let stdout = '';
let stderr = '';
return {
stdout: {
write(chunk: string) {
stdout += chunk;
},
},
stderr: {
write(chunk: string) {
stderr += chunk;
},
},
stdoutText() {
return stdout;
},
stderrText() {
return stderr;
},
};
}
function splitOutputLines(output: string): string[] {
return output
.split('\n')
.map((line) => line.trim())
.filter(Boolean);
}
async function runLowLevelMapping(
args: KloConnectionMappingArgs,
argv: string[],
io: KloCliIo,
deps: KloConnectionDeps,
): Promise<number> {
if (deps.runMapping) {
return await deps.runMapping(argv, io);
}
const { runKloConnectionMapping } = await import('./commands/connection-mapping.js');
return await runKloConnectionMapping(args, io);
}
function parseMappingListJson(output: string): unknown[] {
const trimmed = output.trim();
if (!trimmed) {
return [];
}
const parsed = JSON.parse(trimmed) as unknown;
return Array.isArray(parsed) ? parsed : [];
}
async function runPublicConnectionMap(
args: Extract<KloConnectionArgs, { command: 'map' }>,
io: KloCliIo,
deps: KloConnectionDeps,
): Promise<number> {
const refreshIo = createBufferedIo();
const refreshArgs: KloConnectionMappingArgs = {
command: 'refresh',
projectDir: args.projectDir,
connectionId: args.sourceConnectionId,
autoAccept: true,
};
const refreshCode = await runLowLevelMapping(
refreshArgs,
['refresh', args.sourceConnectionId, '--auto-accept', '--project-dir', args.projectDir],
refreshIo,
deps,
);
if (refreshCode !== 0) {
io.stderr.write(
refreshIo.stderrText() ||
refreshIo.stdoutText() ||
`Failed to refresh mapping metadata for ${args.sourceConnectionId}\n`,
);
return refreshCode;
}
const validationIo = createBufferedIo();
const validationArgs: KloConnectionMappingArgs = {
command: 'validate',
projectDir: args.projectDir,
connectionId: args.sourceConnectionId,
};
const validationCode = await runLowLevelMapping(
validationArgs,
['validate', args.sourceConnectionId, '--project-dir', args.projectDir],
validationIo,
deps,
);
if (validationCode !== 0) {
io.stderr.write(
validationIo.stderrText() || validationIo.stdoutText() || `Mapping validation failed for ${args.sourceConnectionId}\n`,
);
return validationCode;
}
const listIo = createBufferedIo();
const listArgv = ['list', args.sourceConnectionId, '--project-dir', args.projectDir];
const listArgs: KloConnectionMappingArgs = {
command: 'list',
projectDir: args.projectDir,
connectionId: args.sourceConnectionId,
json: args.json,
};
const listCode = await runLowLevelMapping(listArgs, args.json ? [...listArgv, '--json'] : listArgv, listIo, deps);
if (listCode !== 0) {
io.stderr.write(listIo.stderrText() || listIo.stdoutText() || `Failed to list mappings for ${args.sourceConnectionId}\n`);
return listCode;
}
if (args.json) {
io.stdout.write(
`${JSON.stringify(
{
connectionId: args.sourceConnectionId,
refresh: { ok: true, output: splitOutputLines(refreshIo.stdoutText()) },
validation: { ok: true, output: splitOutputLines(validationIo.stdoutText()) },
mappings: parseMappingListJson(listIo.stdoutText()),
},
null,
2,
)}\n`,
);
return 0;
}
io.stdout.write(`Mapping: ${args.sourceConnectionId}\n`);
io.stdout.write(refreshIo.stdoutText());
io.stdout.write(validationIo.stdoutText());
io.stdout.write('\nMappings:\n');
io.stdout.write(listIo.stdoutText().trim() ? listIo.stdoutText() : 'No mappings found.\n');
io.stdout.write('\nNext:\n');
io.stdout.write(` klo ingest ${args.sourceConnectionId}\n`);
io.stdout.write(` klo dev mapping list ${args.sourceConnectionId}\n`);
return 0;
}
export async function runKloConnection(
args: KloConnectionArgs,
io: KloConnectionIo = process,
deps: KloConnectionDeps = {},
): Promise<number> {
try {
if (args.command === 'map') {
return await runPublicConnectionMap(args, io, deps);
}
const project = await loadKloProject({ projectDir: args.projectDir });
if (args.command === 'list') {
const entries = Object.entries(project.config.connections).sort(([a], [b]) => a.localeCompare(b));
if (entries.length === 0) {
io.stdout.write('No connections configured. Run `klo connection add <id> --driver <driver>` to add one.\n');
return 0;
}
const idWidth = Math.max('ID'.length, ...entries.map(([id]) => id.length));
const driverWidth = Math.max(
'DRIVER'.length,
...entries.map(([, c]) => (c.driver ?? 'unknown').length),
);
io.stdout.write(`${'ID'.padEnd(idWidth)} ${'DRIVER'.padEnd(driverWidth)}\n`);
for (const [id, connection] of entries) {
io.stdout.write(`${id.padEnd(idWidth)} ${(connection.driver ?? 'unknown').padEnd(driverWidth)}\n`);
}
return 0;
}
if (args.command === 'add') {
assertSafeConnectionId(args.connectionId);
const hasLiteralCredentialUrl = !!args.url && !isCredentialReference(args.url);
if (hasLiteralCredentialUrl && !args.allowLiteralCredentials) {
throw new Error('Literal credential URLs require --allow-literal-credentials');
}
if (hasLiteralCredentialUrl) {
io.stderr.write(`${literalCredentialWarning(args.connectionId)}\n`);
}
if (project.config.connections[args.connectionId] && !args.force) {
throw new Error(`Connection "${args.connectionId}" already exists; pass --force to replace it`);
}
const connectionConfig =
args.driver === 'notion' && args.notion
? {
driver: 'notion',
auth_token_ref: args.notion.authTokenRef,
crawl_mode: args.notion.crawlMode,
root_page_ids: args.notion.rootPageIds,
root_database_ids: args.notion.rootDatabaseIds,
root_data_source_ids: args.notion.rootDataSourceIds,
...(args.notion.maxPagesPerRun !== undefined ? { max_pages_per_run: args.notion.maxPagesPerRun } : {}),
...(args.notion.maxKnowledgeCreatesPerRun !== undefined
? { max_knowledge_creates_per_run: args.notion.maxKnowledgeCreatesPerRun }
: {}),
...(args.notion.maxKnowledgeUpdatesPerRun !== undefined
? { max_knowledge_updates_per_run: args.notion.maxKnowledgeUpdatesPerRun }
: {}),
}
: {
driver: args.driver,
...(args.url ? { url: args.url } : {}),
...(args.schemas.length > 0 ? { schemas: args.schemas } : {}),
readonly: args.readonly,
};
const nextConfig = {
...project.config,
connections: {
...project.config.connections,
[args.connectionId]: connectionConfig,
},
};
await project.fileStore.writeFile(
'klo.yaml',
serializeKloProjectConfig(nextConfig),
'klo',
'klo@example.com',
`Update KLO connection: ${args.connectionId}`,
);
io.stdout.write(`Connection: ${args.connectionId}\n`);
io.stdout.write(`Driver: ${args.driver}\n`);
return 0;
}
if (args.command === 'remove') {
if (!project.config.connections[args.connectionId]) {
throw new Error(`Connection "${args.connectionId}" is not configured in klo.yaml`);
}
if (!args.force) {
if (!isInteractiveConnectionIo(args, io)) {
throw new Error(
`connection remove ${args.connectionId} requires --force when input is disabled or not interactive`,
);
}
const prompts = deps.prompts ?? createClackConnectionPromptAdapter();
const confirmed = await prompts.confirm({
message: `Remove connection "${args.connectionId}" from klo.yaml? Ingested artifacts will remain in .klo/.`,
initialValue: false,
});
if (!confirmed) {
prompts.cancel('Connection removal cancelled.');
return 1;
}
}
const { [args.connectionId]: _removedConnection, ...connections } = project.config.connections;
const nextConfig = {
...project.config,
connections,
};
await project.fileStore.writeFile(
'klo.yaml',
serializeKloProjectConfig(nextConfig),
'klo',
'klo@example.com',
`Remove KLO connection: ${args.connectionId}`,
);
io.stdout.write('Connection removed from klo.yaml.\n');
io.stdout.write('Ingested artifacts from this connection remain in .klo/. Run klo dev artifacts to inspect.\n');
return 0;
}
const result = await testNativeConnection(
project,
args.connectionId,
deps.createScanConnector ?? createKloCliScanConnector,
);
io.stdout.write(`Connection test passed: ${args.connectionId}\n`);
io.stdout.write(`Driver: ${result.driver}\n`);
io.stdout.write(`Tables: ${result.tableCount}\n`);
return 0;
} catch (error) {
io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
return 1;
}
}

View file

@ -0,0 +1,303 @@
import { buildDefaultKloProjectConfig, type KloProjectConfig } from '@klo/context/project';
import { describe, expect, it, vi } from 'vitest';
import type { KloPublicIngestProject, KloPublicIngestTargetResult } from './public-ingest.js';
import {
extractProgressMessage,
initViewState,
parseIngestSummary,
parseScanSummary,
renderContextBuildView,
runContextBuild,
} from './context-build-view.js';
function makeIo(options: { isTTY?: boolean } = {}) {
let stdout = '';
let stderr = '';
return {
io: {
stdout: {
isTTY: options.isTTY,
write: (chunk: string) => {
stdout += chunk;
},
},
stderr: {
write: (chunk: string) => {
stderr += chunk;
},
},
},
stdout: () => stdout,
stderr: () => stderr,
};
}
function projectWithConnections(connections: KloProjectConfig['connections']): KloPublicIngestProject {
return {
projectDir: '/tmp/project',
config: {
...buildDefaultKloProjectConfig('warehouse'),
connections,
},
};
}
function successResult(connectionId: string, driver: string, operation: 'scan' | 'source-ingest'): KloPublicIngestTargetResult {
return {
connectionId,
driver,
steps: [
{ operation: 'scan', status: operation === 'scan' ? 'done' : 'skipped' },
{ operation: 'source-ingest', status: operation === 'source-ingest' ? 'done' : 'skipped' },
{ operation: 'enrich', status: 'skipped' },
{ operation: 'memory-update', status: operation === 'source-ingest' ? 'done' : 'skipped' },
],
};
}
function failedResult(connectionId: string, driver: string, operation: 'scan' | 'source-ingest'): KloPublicIngestTargetResult {
return {
connectionId,
driver,
steps: [
{ operation: 'scan', status: operation === 'scan' ? 'failed' : 'skipped', detail: `${connectionId} failed at scan.` },
{ operation: 'source-ingest', status: operation === 'source-ingest' ? 'failed' : 'skipped' },
{ operation: 'enrich', status: 'skipped' },
{ operation: 'memory-update', status: 'not-run' },
],
};
}
describe('extractProgressMessage', () => {
it('extracts percentage and message from scan progress', () => {
expect(extractProgressMessage('\r[45%] Scanning tables...')).toBe('[45%] Scanning tables...');
});
it('extracts from permanent progress lines', () => {
expect(extractProgressMessage('[100%] Done\n')).toBe('[100%] Done');
});
it('returns null for non-progress output', () => {
expect(extractProgressMessage('KLO scan completed\n')).toBeNull();
});
});
describe('parseScanSummary', () => {
it('extracts table count from scan output', () => {
expect(parseScanSummary('Semantic layer comparison found 5 changes across 42 tables')).toBe('42 tables');
});
it('handles singular form', () => {
expect(parseScanSummary('found 1 change across 1 table')).toBe('1 tables');
});
it('returns null when no match', () => {
expect(parseScanSummary('No changes detected')).toBeNull();
});
});
describe('parseIngestSummary', () => {
it('extracts work units and saved memory', () => {
expect(parseIngestSummary('Work units: 5\nSaved memory: 3 wiki, 2 SL')).toBe('5 work units · 3 wiki, 2 SL');
});
it('extracts work units alone when no saved memory', () => {
expect(parseIngestSummary('Work units: 5\nStatus: done')).toBe('5 work units');
});
it('extracts saved memory alone when no work units', () => {
expect(parseIngestSummary('Saved memory: 3 wiki, 2 SL')).toBe('3 wiki, 2 SL');
});
it('returns null when no match', () => {
expect(parseIngestSummary('Status: done')).toBeNull();
});
});
describe('initViewState', () => {
it('partitions targets into primary and context sources', () => {
const state = initViewState([
{ connectionId: 'warehouse', driver: 'postgres', operation: 'scan', debugCommand: '', steps: ['scan'] },
{ connectionId: 'dbt-main', driver: 'dbt', operation: 'source-ingest', adapter: 'dbt', debugCommand: '', steps: ['source-ingest', 'memory-update'] },
]);
expect(state.primarySources).toHaveLength(1);
expect(state.primarySources[0].target.connectionId).toBe('warehouse');
expect(state.contextSources).toHaveLength(1);
expect(state.contextSources[0].target.connectionId).toBe('dbt-main');
expect(state.frame).toBe(0);
});
});
describe('renderContextBuildView', () => {
it('renders all-queued state', () => {
const state = initViewState([
{ connectionId: 'warehouse', driver: 'postgres', operation: 'scan', debugCommand: '', steps: ['scan'] },
{ connectionId: 'dbt-main', driver: 'dbt', operation: 'source-ingest', adapter: 'dbt', debugCommand: '', steps: ['source-ingest', 'memory-update'] },
]);
const output = renderContextBuildView(state, { styled: false });
expect(output).toContain('Building KLO context');
expect(output).toContain('Primary sources:');
expect(output).toContain('warehouse');
expect(output).toContain('queued');
expect(output).toContain('Context sources:');
expect(output).toContain('dbt-main');
});
it('renders completed state with summary', () => {
const state = initViewState([
{ connectionId: 'warehouse', driver: 'postgres', operation: 'scan', debugCommand: '', steps: ['scan'] },
]);
state.primarySources[0].status = 'done';
state.primarySources[0].elapsedMs = 72000;
state.primarySources[0].summaryText = '42 tables';
const output = renderContextBuildView(state, { styled: false });
expect(output).toContain('42 tables');
expect(output).toContain('1m12s');
});
it('renders failed state', () => {
const state = initViewState([
{ connectionId: 'warehouse', driver: 'postgres', operation: 'scan', debugCommand: '', steps: ['scan'] },
]);
state.primarySources[0].status = 'failed';
const output = renderContextBuildView(state, { styled: false });
expect(output).toContain('✗');
expect(output).toContain('failed');
});
it('omits empty groups', () => {
const state = initViewState([
{ connectionId: 'dbt-main', driver: 'dbt', operation: 'source-ingest', adapter: 'dbt', debugCommand: '', steps: ['source-ingest', 'memory-update'] },
]);
const output = renderContextBuildView(state, { styled: false });
expect(output).not.toContain('Primary sources:');
expect(output).toContain('Context sources:');
});
});
describe('runContextBuild', () => {
it('executes scan targets before source-ingest targets', async () => {
const io = makeIo();
const project = projectWithConnections({
dbt_main: { driver: 'dbt' },
warehouse: { driver: 'postgres' },
});
const callOrder: string[] = [];
const executeTarget = vi.fn(async (target) => {
callOrder.push(target.connectionId);
return successResult(target.connectionId, target.driver, target.operation);
});
const result = await runContextBuild(
project,
{ projectDir: '/tmp/project', inputMode: 'disabled' },
io.io,
{ executeTarget, now: () => 1000 },
);
expect(result).toEqual({ exitCode: 0, detached: false });
expect(callOrder).toEqual(['warehouse', 'dbt_main']);
});
it('returns exit code 1 when any target fails', async () => {
const io = makeIo();
const project = projectWithConnections({
warehouse: { driver: 'postgres' },
});
const executeTarget = vi.fn(async (target) => failedResult(target.connectionId, target.driver, target.operation));
const result = await runContextBuild(
project,
{ projectDir: '/tmp/project', inputMode: 'disabled' },
io.io,
{ executeTarget, now: () => 1000 },
);
expect(result).toEqual({ exitCode: 1, detached: false });
});
it('renders final view for non-TTY output', async () => {
const io = makeIo();
const project = projectWithConnections({
warehouse: { driver: 'postgres' },
dbt_main: { driver: 'dbt' },
});
const executeTarget = vi.fn(async (target) => successResult(target.connectionId, target.driver, target.operation));
await runContextBuild(
project,
{ projectDir: '/tmp/project', inputMode: 'disabled' },
io.io,
{ executeTarget, now: () => 1000 },
);
const output = io.stdout();
expect(output).toContain('Building KLO context');
expect(output).toContain('Primary sources:');
expect(output).toContain('warehouse');
expect(output).toContain('Context sources:');
expect(output).toContain('dbt_main');
});
it('passes scan mode and detect relationships through to target execution', async () => {
const io = makeIo();
const project = projectWithConnections({ warehouse: { driver: 'postgres' } });
const executeTarget = vi.fn(async (target) => successResult(target.connectionId, target.driver, target.operation));
await runContextBuild(
project,
{ projectDir: '/tmp/project', inputMode: 'disabled', scanMode: 'enriched', detectRelationships: true },
io.io,
{ executeTarget, now: () => 1000 },
);
expect(executeTarget).toHaveBeenCalledWith(
expect.objectContaining({ connectionId: 'warehouse', operation: 'scan' }),
expect.objectContaining({ scanMode: 'enriched', detectRelationships: true }),
expect.anything(),
{},
);
});
it('exits immediately with paused message when d is pressed', async () => {
const mockExit = vi.spyOn(process, 'exit').mockImplementation(() => {
throw new Error('process.exit');
});
const io = makeIo();
const project = projectWithConnections({
warehouse: { driver: 'postgres' },
dbt_main: { driver: 'dbt' },
});
let triggerDetach: (() => void) | null = null;
const executeTarget = vi.fn(async (target) => {
if (target.connectionId === 'warehouse') triggerDetach?.();
return successResult(target.connectionId, target.driver, target.operation);
});
await expect(
runContextBuild(
project,
{ projectDir: '/tmp/project', inputMode: 'disabled' },
io.io,
{
executeTarget,
now: () => 1000,
setupKeystroke: (onDetach) => {
triggerDetach = onDetach;
return () => {};
},
},
),
).rejects.toThrow('process.exit');
expect(mockExit).toHaveBeenCalledWith(0);
expect(io.stdout()).toContain('Context build continuing in the background.');
expect(io.stdout()).toContain('Resume: klo setup --project-dir /tmp/project');
mockExit.mockRestore();
});
});

View file

@ -0,0 +1,414 @@
import { spawn } from 'node:child_process';
import { mkdirSync, openSync } from 'node:fs';
import { join, resolve } from 'node:path';
import type { KloCliIo } from './index.js';
import type {
KloPublicIngestArgs,
KloPublicIngestPlanTarget,
KloPublicIngestProject,
KloPublicIngestTargetResult,
} from './public-ingest.js';
import { buildPublicIngestPlan, executePublicIngestTarget } from './public-ingest.js';
import { formatDuration } from './demo-metrics.js';
import { profileMark } from './startup-profile.js';
profileMark('module:context-build-view');
const SPINNER_FRAMES = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'] as const;
const ESC = String.fromCharCode(0x1b);
export interface ContextBuildTargetState {
target: KloPublicIngestPlanTarget;
status: 'queued' | 'running' | 'done' | 'failed';
detailLine: string | null;
summaryText: string | null;
startedAt: number | null;
elapsedMs: number;
}
export interface ContextBuildViewState {
primarySources: ContextBuildTargetState[];
contextSources: ContextBuildTargetState[];
frame: number;
}
export interface ContextBuildArgs {
projectDir: string;
inputMode: 'auto' | 'disabled';
scanMode?: 'structural' | 'enriched';
detectRelationships?: boolean;
}
export interface ContextBuildResult {
exitCode: number;
detached: boolean;
}
export interface ContextBuildDeps {
executeTarget?: typeof executePublicIngestTarget;
now?: () => number;
setupKeystroke?: (onDetach: () => void, onCtrlC: () => void) => (() => void) | null;
onDetach?: () => void;
}
// --- Rendering ---
function green(text: string): string {
return `${ESC}[32m${text}${ESC}[39m`;
}
function red(text: string): string {
return `${ESC}[31m${text}${ESC}[39m`;
}
function cyan(text: string): string {
return `${ESC}[36m${text}${ESC}[39m`;
}
function dim(text: string): string {
return `${ESC}[2m${text}${ESC}[22m`;
}
function statusIcon(status: ContextBuildTargetState['status'], frame: number, styled: boolean): string {
if (!styled) {
switch (status) {
case 'done':
return '✓';
case 'failed':
return '✗';
case 'running':
return SPINNER_FRAMES[frame % SPINNER_FRAMES.length] ?? '⠋';
default:
return '·';
}
}
switch (status) {
case 'done':
return green('✓');
case 'failed':
return red('✗');
case 'running':
return cyan(SPINNER_FRAMES[frame % SPINNER_FRAMES.length] ?? '⠋');
default:
return dim('·');
}
}
function targetDetail(target: ContextBuildTargetState, styled: boolean): string {
if (target.status === 'done') {
const parts: string[] = [];
if (target.summaryText) parts.push(target.summaryText);
parts.push(formatDuration(target.elapsedMs));
return parts.join(' · ');
}
if (target.status === 'failed') {
return styled ? red('failed') : 'failed';
}
if (target.status === 'running') {
return target.detailLine ?? (target.target.operation === 'scan' ? 'scanning...' : 'ingesting...');
}
return styled ? dim('queued') : 'queued';
}
function columnWidth(state: ContextBuildViewState): number {
const all = [...state.primarySources, ...state.contextSources];
return Math.max(12, ...all.map((t) => t.target.connectionId.length)) + 2;
}
function renderTargetLine(target: ContextBuildTargetState, frame: number, styled: boolean, width: number): string {
return ` ${statusIcon(target.status, frame, styled)} ${target.target.connectionId.padEnd(width)} ${targetDetail(target, styled)}`;
}
function renderTargetGroup(
label: string,
targets: ContextBuildTargetState[],
frame: number,
styled: boolean,
width: number,
): string[] {
if (targets.length === 0) return [];
return ['', ` ${label}:`, ...targets.map((t) => renderTargetLine(t, frame, styled, width))];
}
function resumeCommand(projectDir?: string): string {
return projectDir ? `klo setup --project-dir ${projectDir}` : 'klo setup';
}
export function renderContextBuildView(
state: ContextBuildViewState,
options: { styled?: boolean; showHint?: boolean; projectDir?: string } = {},
): string {
const styled = options.styled ?? true;
const width = columnWidth(state);
const lines: string[] = [
'',
'Building KLO context',
'─────────────────────',
...renderTargetGroup('Primary sources', state.primarySources, state.frame, styled, width),
...renderTargetGroup('Context sources', state.contextSources, state.frame, styled, width),
'',
];
const hasActive = [...state.primarySources, ...state.contextSources].some(
(t) => t.status === 'running' || t.status === 'queued',
);
if (options.showHint && hasActive) {
const hint = ` d to detach · ${resumeCommand(options.projectDir)} to resume`;
lines.push(styled ? dim(hint) : hint);
lines.push('');
}
return `${lines.join('\n')}\n`;
}
// --- IO Capture ---
const ESC_K_RE = new RegExp(`${ESC.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\[K`, 'g');
export function extractProgressMessage(chunk: string): string | null {
const cleaned = chunk.replace(/^\r/, '').replace(ESC_K_RE, '').replace(/\n$/, '').trim();
const match = cleaned.match(/^\[(\d+)%\]\s*(.+)$/);
return match ? `[${match[1]}%] ${match[2]}` : null;
}
export function parseScanSummary(output: string): string | null {
const match = output.match(/(\d+) changes? across (\d+) tables?/);
return match ? `${match[2]} tables` : null;
}
export function parseIngestSummary(output: string): string | null {
const parts: string[] = [];
const workUnits = output.match(/Work units: (\d+)/);
if (workUnits) parts.push(`${workUnits[1]} work units`);
const savedMemory = output.match(/Saved memory: (.+)/);
if (savedMemory) parts.push(savedMemory[1]);
return parts.length > 0 ? parts.join(' · ') : null;
}
interface CapturedIo {
io: KloCliIo;
captured(): string;
}
function createCaptureIo(onProgress: (message: string) => void, isTTY: boolean): CapturedIo {
let buffer = '';
return {
io: {
stdout: {
isTTY,
write(chunk: string) {
buffer += chunk;
const progress = extractProgressMessage(chunk);
if (progress) onProgress(progress);
},
},
stderr: {
write(chunk: string) {
buffer += chunk;
},
},
},
captured: () => buffer,
};
}
// --- Repaint ---
function createRepainter(io: KloCliIo) {
let lastLineCount = 0;
return {
paint(content: string) {
if (lastLineCount > 0) {
io.stdout.write(`${ESC}[${lastLineCount}A\r`);
}
io.stdout.write(content);
io.stdout.write(`${ESC}[J`);
lastLineCount = (content.match(/\n/g) ?? []).length;
},
};
}
// --- Background build ---
function resolveKloEntryScript(): string | null {
const argv1 = process.argv[1];
if (argv1 && (argv1.endsWith('.js') || argv1.endsWith('.ts') || argv1.endsWith('.mjs'))) {
return argv1;
}
return null;
}
function spawnBackgroundBuild(projectDir: string): { logPath: string } | null {
const entryScript = resolveKloEntryScript();
if (!entryScript) return null;
const resolvedDir = resolve(projectDir);
const logDir = join(resolvedDir, '.klo', 'setup');
mkdirSync(logDir, { recursive: true });
const logPath = join(logDir, 'context-build.log');
const logFd = openSync(logPath, 'w');
const child = spawn(
process.execPath,
[entryScript, 'setup', 'context', 'build', '--project-dir', resolvedDir, '--no-input'],
{ detached: true, stdio: ['ignore', logFd, logFd] },
);
child.unref();
return { logPath };
}
// --- Keystroke handling ---
function defaultSetupKeystroke(onDetach: () => void, onCtrlC: () => void): (() => void) | null {
const stdin = process.stdin;
if (!stdin.isTTY || typeof stdin.setRawMode !== 'function') {
return null;
}
stdin.setRawMode(true);
stdin.resume();
const onData = (data: Buffer) => {
const char = data.toString();
if (char === 'd' || char === 'D') onDetach();
else if (char === '\x03') onCtrlC();
};
stdin.on('data', onData);
return () => {
stdin.off('data', onData);
if (typeof stdin.setRawMode === 'function') stdin.setRawMode(false);
stdin.pause();
};
}
// --- Orchestration ---
function makeTargetState(target: KloPublicIngestPlanTarget): ContextBuildTargetState {
return { target, status: 'queued', detailLine: null, summaryText: null, startedAt: null, elapsedMs: 0 };
}
export function initViewState(targets: KloPublicIngestPlanTarget[]): ContextBuildViewState {
return {
primarySources: targets.filter((t) => t.operation === 'scan').map(makeTargetState),
contextSources: targets.filter((t) => t.operation === 'source-ingest').map(makeTargetState),
frame: 0,
};
}
export async function runContextBuild(
project: KloPublicIngestProject,
args: ContextBuildArgs,
io: KloCliIo,
deps: ContextBuildDeps = {},
): Promise<ContextBuildResult> {
const plan = buildPublicIngestPlan(project, { projectDir: args.projectDir, all: true });
const state = initViewState(plan.targets);
const isTTY = io.stdout.isTTY === true;
const nowFn = deps.now ?? (() => Date.now());
const repainter = isTTY ? createRepainter(io) : null;
const viewOpts = { styled: true, projectDir: args.projectDir };
const paint = (hint: boolean) => repainter?.paint(renderContextBuildView(state, { ...viewOpts, showHint: hint }));
paint(true);
let spinnerInterval: ReturnType<typeof setInterval> | null = null;
if (repainter) {
spinnerInterval = setInterval(() => {
state.frame++;
for (const t of [...state.primarySources, ...state.contextSources]) {
if (t.status === 'running' && t.startedAt !== null) {
t.elapsedMs = nowFn() - t.startedAt;
}
}
paint(true);
}, 140);
}
const orderedTargets = [...state.primarySources, ...state.contextSources];
const execTarget = deps.executeTarget ?? executePublicIngestTarget;
let detached = false;
let cleanupKeystroke: (() => void) | null = null;
if (isTTY || deps.setupKeystroke) {
const cleanup = () => {
if (spinnerInterval) clearInterval(spinnerInterval);
cleanupKeystroke?.();
};
cleanupKeystroke = (deps.setupKeystroke ?? defaultSetupKeystroke)(
() => {
cleanup();
deps.onDetach?.();
const bg = spawnBackgroundBuild(args.projectDir);
io.stdout.write('\n\nContext build continuing in the background.\n');
if (bg) io.stdout.write(`Log: ${bg.logPath}\n`);
io.stdout.write(`Status: klo setup context status --project-dir ${resolve(args.projectDir)}\n`);
io.stdout.write(`Resume: ${resumeCommand(args.projectDir)}\n`);
process.exit(0);
},
() => {
cleanup();
io.stdout.write('\n\nContext build stopped. Nothing is running in the background.\n');
io.stdout.write(`Resume: ${resumeCommand(args.projectDir)}\n`);
process.exit(130);
},
);
}
const runArgs: Extract<KloPublicIngestArgs, { command: 'run' }> = {
command: 'run',
projectDir: args.projectDir,
all: true,
json: false,
inputMode: args.inputMode,
scanMode: args.scanMode,
detectRelationships: args.detectRelationships,
};
let hasFailure = false;
try {
for (const targetState of orderedTargets) {
if (detached) break;
targetState.status = 'running';
targetState.startedAt = nowFn();
paint(true);
const capture = createCaptureIo(
(message) => {
targetState.detailLine = message;
paint(true);
},
false,
);
const result = await execTarget(targetState.target, runArgs, capture.io, {});
targetState.elapsedMs = nowFn() - (targetState.startedAt ?? nowFn());
const failed = result.steps.some((s) => s.status === 'failed');
targetState.status = failed ? 'failed' : 'done';
targetState.detailLine = null;
if (!failed) {
targetState.summaryText =
targetState.target.operation === 'scan'
? parseScanSummary(capture.captured())
: parseIngestSummary(capture.captured());
}
if (failed) hasFailure = true;
paint(true);
}
} finally {
if (spinnerInterval) clearInterval(spinnerInterval);
cleanupKeystroke?.();
}
if (detached) {
return { exitCode: 0, detached: true };
}
if (!repainter) {
io.stdout.write(renderContextBuildView(state, { styled: false }));
} else {
paint(false);
}
return { exitCode: hasFailure ? 1 : 0, detached: false };
}

View file

@ -0,0 +1,272 @@
import { access, readFile, rm, stat, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { afterEach, describe, expect, it } from 'vitest';
import {
DEMO_ADAPTER,
DEMO_CONNECTION_ID,
DEMO_FULL_JOB_ID,
DEMO_REPLAY_FILE,
defaultDemoProjectDir,
ensureDemoProject,
inspectDemoProjectState,
loadPackagedDemoReplay,
loadProjectDemoReplay,
resetDemoProject,
} from './demo-assets.js';
import { writeDemoReplay } from './demo-replay-store.js';
const packagedDemoSource = 'packaged-orbit-demo';
function packagedDemoAssetPath(relativePath: string): string {
return fileURLToPath(new URL(`../assets/demo/orbit/${relativePath}`, import.meta.url));
}
async function readPackagedJson<T>(relativePath: string): Promise<T> {
return JSON.parse(await readFile(packagedDemoAssetPath(relativePath), 'utf-8')) as T;
}
describe('demo assets', () => {
const projectDir = join(tmpdir(), `klo-demo-assets-${process.pid}`);
afterEach(async () => {
await rm(projectDir, { recursive: true, force: true });
});
it('resolves the default demo root under the OS temp directory', () => {
const dir = defaultDemoProjectDir();
expect(dir.startsWith(join(tmpdir(), 'klo-demo-'))).toBe(true);
expect(dir).toMatch(/klo-demo-[a-f0-9]{8}$/);
});
it('exports the packaged Orbit demo identity', () => {
expect(DEMO_CONNECTION_ID).toBe('orbit_demo');
expect(DEMO_ADAPTER).toBe('live-database');
expect(DEMO_REPLAY_FILE).toBe('replay.memory-flow.v1.json');
expect(DEMO_FULL_JOB_ID).toBe('demo-full-ingest');
});
it('ships the seeded demo bundle required by the May 6 PRD', async () => {
const manifest = await readPackagedJson<{
demoAssetSchemaVersion: number;
mode: string;
source: string;
sources: {
warehouse: { tables: number; rowCounts: Record<string, number> };
dbt: { models: number; sourceTables: number };
bi: { explores: number; dashboards: number };
notion: { pages: number };
};
name: string;
displayName: string;
generated: {
semanticLayer: { path: string; sourceCount: number };
knowledge: { pageCount: number };
links: { linkCount: number };
};
}>('manifest.json');
expect(manifest).toMatchObject({
demoAssetSchemaVersion: 2,
name: 'orbit',
displayName: 'Orbit Demo',
mode: 'seeded',
source: packagedDemoSource,
});
expect(manifest.sources.warehouse.tables).toBeGreaterThanOrEqual(5);
expect(manifest.sources.warehouse.tables).toBeLessThanOrEqual(10);
expect(Object.keys(manifest.sources.warehouse.rowCounts).sort()).toEqual([
'accounts',
'arr_movements',
'contracts',
'invoices',
'plans',
'purchase_requests',
'support_tickets',
'users',
]);
expect(manifest.sources.dbt.models).toBeGreaterThanOrEqual(3);
expect(manifest.sources.dbt.models).toBeLessThanOrEqual(6);
expect(manifest.sources.bi.explores).toBeGreaterThanOrEqual(2);
expect(manifest.sources.bi.dashboards).toBeGreaterThanOrEqual(2);
expect(manifest.sources.notion.pages).toBeGreaterThanOrEqual(5);
expect(manifest.generated.semanticLayer.sourceCount).toBeGreaterThanOrEqual(5);
expect(manifest.generated.knowledge.pageCount).toBeGreaterThanOrEqual(10);
expect(manifest.generated.links.linkCount).toBeGreaterThanOrEqual(10);
const dbStat = await stat(packagedDemoAssetPath('demo.db'));
expect(dbStat.size).toBeGreaterThan(0);
expect(dbStat.size).toBeLessThan(10 * 1024 * 1024);
await expect(access(packagedDemoAssetPath('raw-sources/warehouse/accounts.csv'))).resolves.toBeUndefined();
await expect(access(packagedDemoAssetPath('raw-sources/dbt/schema.yml'))).resolves.toBeUndefined();
await expect(access(packagedDemoAssetPath('raw-sources/bi/revenue_exec.dashboard.lookml'))).resolves.toBeUndefined();
await expect(access(packagedDemoAssetPath('raw-sources/notion/revenue-reporting-policy.md'))).resolves.toBeUndefined();
expect(manifest.generated.semanticLayer.path).toBe('semantic-layer/orbit_demo');
await expect(access(packagedDemoAssetPath('semantic-layer/orbit_demo/accounts.yaml'))).resolves.toBeUndefined();
await expect(access(packagedDemoAssetPath('knowledge/global/arr-contract-first.md'))).resolves.toBeUndefined();
await expect(access(packagedDemoAssetPath('links/provenance.json'))).resolves.toBeUndefined();
await expect(access(packagedDemoAssetPath('reports/seeded-demo-report.json'))).resolves.toBeUndefined();
});
it('initializes a flat demo project without writing literal credentials', async () => {
const result = await ensureDemoProject({ projectDir, force: false });
expect(result.projectDir).toBe(projectDir);
await expect(access(join(projectDir, 'demo.db'))).resolves.toBeUndefined();
await expect(access(join(projectDir, 'state.sqlite'))).resolves.toBeUndefined();
await expect(access(join(projectDir, 'reports'))).resolves.toBeUndefined();
await expect(access(join(projectDir, 'semantic-layer'))).resolves.toBeUndefined();
await expect(access(join(projectDir, 'knowledge'))).resolves.toBeUndefined();
await expect(access(join(projectDir, 'replays', 'replay.memory-flow.v1.json'))).resolves.toBeUndefined();
await expect(access(join(projectDir, 'raw-sources'))).resolves.toBeUndefined();
await expect(access(join(projectDir, '_schema'))).rejects.toMatchObject({ code: 'ENOENT' });
const config = await readFile(join(projectDir, 'klo.yaml'), 'utf-8');
expect(config).toContain('backend: anthropic');
expect(config).toContain('api_key: env:ANTHROPIC_API_KEY');
expect(config).not.toContain('sk-ant-');
});
it('rejects an existing demo project unless force is set', async () => {
await ensureDemoProject({ projectDir, force: false });
await expect(ensureDemoProject({ projectDir, force: false })).rejects.toThrow('Demo project already exists');
await expect(ensureDemoProject({ projectDir, force: true })).resolves.toMatchObject({ projectDir });
});
it('loads packaged and copied demo replays', async () => {
const packaged = await loadPackagedDemoReplay();
expect(packaged.runId).toBe('demo-seeded-orbit');
expect(packaged.connectionId).toBe('orbit_demo');
expect(packaged.metadata?.mode).toBe('seeded');
await ensureDemoProject({ projectDir, force: false });
const copied = await loadProjectDemoReplay(projectDir);
expect(copied).toEqual(packaged);
});
it('loads the latest local replay before the packaged replay', async () => {
await ensureDemoProject({ projectDir, force: false });
await writeDemoReplay(
projectDir,
{
metadata: {
schemaVersion: 1,
mode: 'full',
origin: 'captured',
timing: 'captured',
capturedAt: '2026-05-01T10:00:03.000Z',
sourceReportId: null,
sourceReportPath: 'raw-sources/orbit_demo/live-database/sync/scan-report.json',
fallbackReason: null,
},
runId: 'demo-full-run',
connectionId: 'orbit_demo',
adapter: 'live-database',
status: 'done',
sourceDir: null,
syncId: 'sync',
reportPath: 'raw-sources/orbit_demo/live-database/sync/scan-report.json',
errors: [],
events: [{ type: 'report_created', runId: 'scan-run' }],
plannedWorkUnits: [],
details: { actions: [], provenance: [], transcripts: [] },
},
{ label: 'full' },
);
await expect(loadProjectDemoReplay(projectDir)).resolves.toMatchObject({
runId: 'demo-full-run',
metadata: { mode: 'full', origin: 'captured' },
});
});
it('reports missing, ready, and corrupted demo project state', async () => {
await expect(inspectDemoProjectState(projectDir)).resolves.toEqual({
status: 'missing',
projectDir,
missing: ['klo.yaml', 'demo.db', 'state.sqlite', 'replays/replay.memory-flow.v1.json'],
});
await ensureDemoProject({ projectDir, force: false });
await expect(inspectDemoProjectState(projectDir)).resolves.toEqual({
status: 'ready',
projectDir,
missing: [],
});
await rm(join(projectDir, 'demo.db'), { force: true });
await expect(inspectDemoProjectState(projectDir)).resolves.toEqual({
status: 'corrupt',
projectDir,
missing: ['demo.db'],
});
});
it('requires explicit force for demo reset and recreates packaged assets', async () => {
await ensureDemoProject({ projectDir, force: false });
await rm(join(projectDir, 'demo.db'), { force: true });
await expect(resetDemoProject({ projectDir, force: false })).rejects.toThrow(
`klo setup demo reset is destructive; pass --force to recreate ${projectDir}`,
);
await expect(resetDemoProject({ projectDir, force: true })).resolves.toMatchObject({ projectDir });
await expect(access(join(projectDir, 'demo.db'))).resolves.toBeUndefined();
await expect(inspectDemoProjectState(projectDir)).resolves.toMatchObject({ status: 'ready' });
});
it('preserves a user-edited klo.yaml across reset --force', async () => {
await ensureDemoProject({ projectDir, force: false });
const customConfig = [
'project: klo-demo-orbit',
'connections:',
` ${DEMO_CONNECTION_ID}:`,
' driver: sqlite',
` path: ${JSON.stringify(join(projectDir, 'demo.db'))}`,
' readonly: true',
'storage:',
' state: sqlite',
' search: sqlite-fts5',
' git:',
' auto_commit: true',
' author: klo <klo@example.com>',
'llm:',
' provider:',
' backend: vertex',
' vertex:',
' project: example-gcp-project',
' location: us-east5',
' models:',
' default: claude-sonnet-4-6',
'ingest:',
' adapters:',
` - ${DEMO_ADAPTER}`,
' embeddings:',
' backend: none',
' dimensions: 8',
' workUnits:',
' stepBudget: 40',
' maxConcurrency: 1',
' failureMode: continue',
'',
].join('\n');
await writeFile(join(projectDir, 'klo.yaml'), customConfig, 'utf-8');
await resetDemoProject({ projectDir, force: true });
const preserved = await readFile(join(projectDir, 'klo.yaml'), 'utf-8');
expect(preserved).toBe(customConfig);
expect(preserved).toContain('backend: vertex');
expect(preserved).not.toContain('backend: anthropic');
await expect(inspectDemoProjectState(projectDir)).resolves.toMatchObject({ status: 'ready' });
});
it('still writes the default klo.yaml on reset when none exists', async () => {
await resetDemoProject({ projectDir, force: true });
const config = await readFile(join(projectDir, 'klo.yaml'), 'utf-8');
expect(config).toContain('backend: anthropic');
});
});

View file

@ -0,0 +1,281 @@
import { constants as fsConstants } from 'node:fs';
import { access, copyFile, cp, mkdir, readFile, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { dirname, join, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import { randomBytes } from 'node:crypto';
import type { MemoryFlowReplayInput } from '@klo/context/ingest/memory-flow';
import { loadDemoReplayFile, loadLatestDemoReplay } from './demo-replay-store.js';
interface DemoProjectResult {
projectDir: string;
configPath: string;
databasePath: string;
replayPath: string;
}
interface EnsureDemoProjectOptions {
projectDir: string;
force: boolean;
}
type DemoProjectStateStatus = 'missing' | 'ready' | 'corrupt';
interface DemoProjectState {
status: DemoProjectStateStatus;
projectDir: string;
missing: string[];
}
export const DEMO_CONNECTION_ID = 'orbit_demo';
export const DEMO_ADAPTER = 'live-database';
export const DEMO_REPLAY_FILE = 'replay.memory-flow.v1.json';
export const DEMO_FULL_JOB_ID = 'demo-full-ingest';
const REQUIRED_BASE_PROJECT_PATHS = [
'klo.yaml',
'demo.db',
'state.sqlite',
join('replays', DEMO_REPLAY_FILE),
] as const;
const REQUIRED_PACKAGED_BASE_ASSET_PATHS = ['demo.db', 'manifest.json', DEMO_REPLAY_FILE] as const;
const REQUIRED_SEEDED_ASSET_PATHS = [
'demo.db',
'manifest.json',
DEMO_REPLAY_FILE,
join('raw-sources', 'warehouse', 'accounts.csv'),
join('raw-sources', 'dbt', 'schema.yml'),
join('raw-sources', 'bi', 'revenue_exec.dashboard.lookml'),
join('raw-sources', 'notion', 'revenue-reporting-policy.md'),
join('semantic-layer', 'orbit_demo', 'accounts.yaml'),
join('knowledge', 'global', 'arr-contract-first.md'),
join('links', 'provenance.json'),
join('reports', 'seeded-demo-report.json'),
] as const;
function assetDir(): string {
return fileURLToPath(new URL('../assets/demo/orbit/', import.meta.url));
}
async function exists(path: string): Promise<boolean> {
try {
await access(path, fsConstants.F_OK);
return true;
} catch {
return false;
}
}
export function defaultDemoProjectDir(): string {
const suffix = randomBytes(4).toString('hex');
return join(tmpdir(), `klo-demo-${suffix}`);
}
export async function inspectDemoProjectState(projectDir: string): Promise<DemoProjectState> {
const root = resolve(projectDir);
const missing: string[] = [];
for (const relativePath of REQUIRED_BASE_PROJECT_PATHS) {
if (!(await exists(join(root, relativePath)))) {
missing.push(relativePath);
}
}
if (missing.length === REQUIRED_BASE_PROJECT_PATHS.length) {
return { status: 'missing', projectDir: root, missing };
}
if (missing.length > 0) {
return { status: 'corrupt', projectDir: root, missing };
}
return { status: 'ready', projectDir: root, missing: [] };
}
export async function resetDemoProject(options: EnsureDemoProjectOptions): Promise<DemoProjectResult> {
const projectDir = resolve(options.projectDir);
if (!options.force) {
throw new Error(`klo setup demo reset is destructive; pass --force to recreate ${projectDir}`);
}
const preservedConfig = await readExistingConfig(join(projectDir, 'klo.yaml'));
const result = await ensureDemoProject({ projectDir, force: true });
if (preservedConfig !== null) {
await writeFile(result.configPath, preservedConfig, 'utf-8');
}
return result;
}
async function readExistingConfig(configPath: string): Promise<string | null> {
try {
return await readFile(configPath, 'utf-8');
} catch {
return null;
}
}
function demoConfig(databasePath: string): string {
return [
'project: klo-demo-orbit',
'connections:',
` ${DEMO_CONNECTION_ID}:`,
' driver: sqlite',
` path: ${JSON.stringify(databasePath)}`,
' readonly: true',
'storage:',
' state: sqlite',
' search: sqlite-fts5',
' git:',
' auto_commit: true',
' author: klo <klo@example.com>',
'llm:',
' provider:',
' backend: anthropic',
' anthropic:',
' api_key: env:ANTHROPIC_API_KEY',
' models:',
' default: claude-sonnet-4-6',
'ingest:',
' adapters:',
` - ${DEMO_ADAPTER}`,
' embeddings:',
' backend: none',
' dimensions: 8',
' workUnits:',
' stepBudget: 40',
' maxConcurrency: 1',
' failureMode: continue',
'',
].join('\n');
}
async function copyPackagedReplay(projectDir: string): Promise<string> {
const replayDir = join(projectDir, 'replays');
await mkdir(replayDir, { recursive: true });
const replayPath = join(replayDir, DEMO_REPLAY_FILE);
await copyFile(join(assetDir(), DEMO_REPLAY_FILE), replayPath);
return replayPath;
}
async function assertPackagedBaseAssetsPresent(): Promise<void> {
const missing: string[] = [];
for (const relativePath of REQUIRED_PACKAGED_BASE_ASSET_PATHS) {
if (!(await exists(join(assetDir(), relativePath)))) {
missing.push(relativePath);
}
}
if (missing.length > 0) {
throw new Error(`Packaged demo assets are incomplete: missing ${missing.join(', ')}`);
}
}
async function assertPackagedSeededAssetsPresent(): Promise<void> {
const missing: string[] = [];
for (const relativePath of REQUIRED_SEEDED_ASSET_PATHS) {
if (!(await exists(join(assetDir(), relativePath)))) {
missing.push(relativePath);
}
}
if (missing.length > 0) {
throw new Error(`Packaged seeded demo assets are incomplete: missing ${missing.join(', ')}`);
}
}
export async function ensureDemoProject(options: EnsureDemoProjectOptions): Promise<DemoProjectResult> {
const projectDir = resolve(options.projectDir);
const configPath = join(projectDir, 'klo.yaml');
if (!options.force && (await exists(configPath))) {
throw new Error(`Demo project already exists at ${projectDir}; pass --force to recreate it`);
}
await assertPackagedBaseAssetsPresent();
if (options.force) {
await rm(projectDir, { recursive: true, force: true });
}
await mkdir(projectDir, { recursive: true });
for (const relativeDir of ['reports', 'semantic-layer', 'knowledge', 'replays', 'raw-sources', 'links']) {
await mkdir(join(projectDir, relativeDir), { recursive: true });
}
const databasePath = join(projectDir, 'demo.db');
await copyFile(join(assetDir(), 'demo.db'), databasePath);
await writeFile(join(projectDir, 'state.sqlite'), '', { flag: 'a' });
await copyFile(join(assetDir(), 'manifest.json'), join(projectDir, 'manifest.json'));
const replayPath = await copyPackagedReplay(projectDir);
await writeFile(configPath, demoConfig(databasePath), 'utf-8');
return { projectDir, configPath, databasePath, replayPath };
}
async function copyDirIfExists(src: string, dest: string): Promise<void> {
if (await exists(src)) {
await cp(src, dest, { recursive: true });
}
}
async function copySeededAssetDirectories(projectDir: string): Promise<void> {
const src = assetDir();
const dest = resolve(projectDir);
await Promise.all([
copyDirIfExists(join(src, 'semantic-layer'), join(dest, 'semantic-layer')),
copyDirIfExists(join(src, 'knowledge'), join(dest, 'knowledge')),
copyDirIfExists(join(src, 'raw-sources'), join(dest, 'raw-sources')),
copyDirIfExists(join(src, 'links'), join(dest, 'links')),
copyDirIfExists(join(src, 'reports'), join(dest, 'reports')),
]);
}
export async function ensureSeededDemoProject(options: EnsureDemoProjectOptions): Promise<DemoProjectResult> {
await assertPackagedSeededAssetsPresent();
const projectDir = resolve(options.projectDir);
const result = await ensureDemoProject(options).catch((error) => {
if (!options.force && error instanceof Error && error.message.includes('Demo project already exists')) {
return {
projectDir,
configPath: join(projectDir, 'klo.yaml'),
databasePath: join(projectDir, 'demo.db'),
replayPath: join(projectDir, 'replays', DEMO_REPLAY_FILE),
};
}
throw error;
});
await copySeededAssetDirectories(result.projectDir);
return result;
}
export async function loadPackagedDemoReplay(): Promise<MemoryFlowReplayInput> {
const replay = await loadDemoReplayFile(join(assetDir(), DEMO_REPLAY_FILE));
return {
...replay,
metadata: {
schemaVersion: 1,
mode: replay.metadata?.mode ?? 'seeded',
origin: 'packaged',
timing: replay.metadata?.timing ?? 'prebuilt',
capturedAt: replay.metadata?.capturedAt ?? null,
sourceReportId: replay.metadata?.sourceReportId ?? 'demo-seeded-report',
sourceReportPath: replay.metadata?.sourceReportPath ?? `reports/seeded-demo-report.json`,
fallbackReason: null,
},
};
}
export async function loadProjectDemoReplay(projectDir: string): Promise<MemoryFlowReplayInput> {
const latest = await loadLatestDemoReplay(projectDir);
if (latest) {
return latest;
}
const replayPath = join(resolve(projectDir), 'replays', DEMO_REPLAY_FILE);
if (!(await exists(replayPath))) {
await mkdir(dirname(replayPath), { recursive: true });
await copyPackagedReplay(resolve(projectDir));
}
return loadPackagedDemoReplay();
}

View file

@ -0,0 +1,201 @@
import { mkdtemp, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import type { IngestReportSnapshot, LocalIngestResult, RunLocalIngestOptions } from '@klo/context/ingest';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { DEMO_ADAPTER, DEMO_CONNECTION_ID, DEMO_FULL_JOB_ID, ensureDemoProject } from './demo-assets.js';
import {
assertFullDemoCredentials,
buildFullDemoReplay,
formatFullDemoSummary,
fullDemoCredentialStatus,
runDemoFull,
} from './demo-full.js';
function fakeFullReport(): IngestReportSnapshot {
return {
id: 'report-full',
runId: 'run-full',
jobId: DEMO_FULL_JOB_ID,
connectionId: DEMO_CONNECTION_ID,
sourceKey: DEMO_ADAPTER,
createdAt: '2026-05-01T00:00:00.000Z',
body: {
syncId: 'sync-full',
diffSummary: { added: 7, modified: 0, deleted: 0, unchanged: 0 },
commitSha: null,
workUnits: [
{
unitKey: 'accounts',
rawFiles: ['accounts.schema.json'],
status: 'success',
actions: [
{ target: 'wiki', type: 'created', key: 'knowledge/accounts.md', detail: 'account lifecycle context' },
{ target: 'sl', type: 'created', key: 'orbit_demo.accounts', detail: 'accounts semantic source' },
],
touchedSlSources: [{ connectionId: 'orbit_demo', sourceName: 'orbit_demo.accounts' }],
},
],
failedWorkUnits: [],
reconciliationSkipped: false,
conflictsResolved: [],
evictionsApplied: [],
unmappedFallbacks: [],
evictionInputs: [],
unresolvedCards: [],
supersededBy: null,
overrideOf: null,
provenanceRows: [
{
rawPath: 'accounts.schema.json',
artifactKind: 'wiki',
artifactKey: 'knowledge/accounts.md',
actionType: 'wiki_written',
},
{
rawPath: 'accounts.schema.json',
artifactKind: 'sl',
artifactKey: 'orbit_demo.accounts',
actionType: 'source_created',
},
],
toolTranscripts: [],
},
};
}
describe('full demo helpers', () => {
let tempDir: string;
let projectDir: string;
beforeEach(async () => {
tempDir = await mkdtemp(join(tmpdir(), 'klo-demo-full-'));
projectDir = join(tempDir, 'demo');
await ensureDemoProject({ projectDir, force: false });
});
afterEach(async () => {
await rm(tempDir, { recursive: true, force: true });
});
it('fails full mode with exact Anthropic env guidance when the key is missing', async () => {
const project = await import('@klo/context/project').then((mod) => mod.loadKloProject({ projectDir }));
expect(() => assertFullDemoCredentials(project, {})).toThrow(
'klo setup demo --mode full needs ANTHROPIC_API_KEY. Export ANTHROPIC_API_KEY and rerun `klo setup demo --mode full --no-input`, or run `klo setup demo --mode seeded --no-input` without credentials.',
);
});
it('respects an existing gateway provider project for full mode', async () => {
await writeFile(
join(projectDir, 'klo.yaml'),
[
'project: klo-demo-orbit',
'connections:',
' orbit_demo:',
' driver: sqlite',
` path: ${JSON.stringify(join(projectDir, 'demo.db'))}`,
'llm:',
' provider:',
' backend: gateway',
' models:',
' default: anthropic/claude-sonnet-4-6',
'ingest:',
' adapters:',
' - live-database',
' embeddings:',
' backend: none',
'',
].join('\n'),
'utf-8',
);
const project = await import('@klo/context/project').then((mod) => mod.loadKloProject({ projectDir }));
expect(() => assertFullDemoCredentials(project, {})).not.toThrow();
expect(fullDemoCredentialStatus(project, {})).toEqual({ status: 'ready' });
});
it('reports full-demo credential status without throwing', async () => {
const project = await import('@klo/context/project').then((mod) => mod.loadKloProject({ projectDir }));
expect(fullDemoCredentialStatus(project, {})).toEqual({ status: 'missing-anthropic-key' });
expect(fullDemoCredentialStatus(project, { ANTHROPIC_API_KEY: 'sk-ant-test' })).toEqual({ status: 'ready' }); // pragma: allowlist secret
await writeFile(
join(projectDir, 'klo.yaml'),
[
'project: klo-demo-orbit',
'connections:',
' orbit_demo:',
' driver: sqlite',
` path: ${JSON.stringify(join(projectDir, 'demo.db'))}`,
'ingest:',
' adapters:',
' - live-database',
'',
].join('\n'),
'utf-8',
);
const disabledProject = await import('@klo/context/project').then((mod) => mod.loadKloProject({ projectDir }));
expect(fullDemoCredentialStatus(disabledProject, {})).toEqual({ status: 'unsupported-provider', provider: 'none' });
});
it('runs scan first and then full ingest with the canonical demo connection', async () => {
const report = fakeFullReport();
const runLocalScan = vi.fn().mockResolvedValue({
report: {
runId: 'scan-run',
connectionId: DEMO_CONNECTION_ID,
driver: 'sqlite',
mode: 'structural',
syncId: 'sync-scan',
diffSummary: { tablesAdded: 7, tablesModified: 0, tablesDeleted: 0, tablesUnchanged: 0 },
artifactPaths: { rawSourcesDir: 'raw-sources/orbit_demo/live-database/sync-scan', manifestShards: [], reportPath: 'scan-report.json' },
},
});
const runLocalIngest = vi.fn(async (options: RunLocalIngestOptions): Promise<LocalIngestResult> => {
expect(options.adapter).toBe(DEMO_ADAPTER);
expect(options.connectionId).toBe(DEMO_CONNECTION_ID);
expect(options.jobId).toBe(DEMO_FULL_JOB_ID);
expect(options.memoryFlow?.snapshot()).toMatchObject({ runId: DEMO_FULL_JOB_ID, status: 'running' });
options.memoryFlow?.emit({ type: 'source_acquired', adapter: DEMO_ADAPTER, trigger: 'demo_full', fileCount: 7 });
return { result: { ok: true } as never, report };
});
const snapshots: unknown[] = [];
const result = await runDemoFull({
projectDir,
env: { ANTHROPIC_API_KEY: 'sk-ant-test' }, // pragma: allowlist secret
runLocalScan,
runLocalIngest,
onMemoryFlowChange: (snapshot) => snapshots.push(snapshot),
});
expect(runLocalScan).toHaveBeenCalledTimes(1);
expect(runLocalIngest).toHaveBeenCalledTimes(1);
expect(result.report).toBe(report);
expect(result.replay.runId).toBe('run-full');
expect(snapshots).toHaveLength(1);
});
it('builds replay and plain summary from the full report', () => {
const report = fakeFullReport();
const replay = buildFullDemoReplay(report);
const summary = formatFullDemoSummary(report);
expect(replay).toMatchObject({
runId: 'run-full',
connectionId: DEMO_CONNECTION_ID,
adapter: DEMO_ADAPTER,
status: 'done',
});
expect(summary).toContain('Full demo ingest: done');
expect(summary).toContain('Saved memory: 1 wiki, 1 semantic layer');
expect(summary).toContain('Provenance rows: 2');
expect(summary).toContain('Next: klo setup demo inspect');
expect(summary).toContain('Shows the files, semantic-layer sources, and memory KLO just produced.');
expect(summary).toContain('Next: klo setup demo replay');
expect(summary).toContain('Replays the same visual story without calling the LLM again.');
expect(summary).not.toContain('--viz');
});
});

View file

@ -0,0 +1,213 @@
import { resolveKloConfigReference } from '@klo/context/core';
import {
createMemoryFlowLiveBuffer,
ingestReportToMemoryFlowReplay,
runLocalIngest,
type IngestReportSnapshot,
type LocalIngestResult,
type MemoryFlowReplayInput,
type RunLocalIngestOptions,
} from '@klo/context/ingest';
import { loadKloProject, type KloLocalProject } from '@klo/context/project';
import { runLocalScan, type LocalScanRunResult } from '@klo/context/scan';
import { DEMO_ADAPTER, DEMO_CONNECTION_ID, DEMO_FULL_JOB_ID, ensureDemoProject } from './demo-assets.js';
import { runDemoScan } from './demo-scan.js';
import { createKloCliLocalIngestAdapters } from './local-adapters.js';
import { formatNextStepLines } from './next-steps.js';
interface DemoFullOptions {
projectDir: string;
env?: NodeJS.ProcessEnv;
runLocalScan?: typeof runLocalScan;
runLocalIngest?: typeof runLocalIngest;
onMemoryFlowChange?: (snapshot: MemoryFlowReplayInput) => void;
}
export interface DemoFullResult {
project: KloLocalProject;
scan: LocalScanRunResult;
ingest: LocalIngestResult;
report: IngestReportSnapshot;
replay: MemoryFlowReplayInput;
}
type FullDemoCredentialStatus =
| { status: 'ready' }
| { status: 'missing-anthropic-key' }
| { status: 'unsupported-provider'; provider: string };
async function ensureDemoProjectForReuse(projectDir: string): Promise<void> {
await ensureDemoProject({ projectDir, force: false }).catch((error) => {
if (error instanceof Error && error.message.includes('Demo project already exists')) {
return;
}
throw error;
});
}
function savedCounts(report: IngestReportSnapshot): { wikiCount: number; slCount: number } {
const actions = report.body.workUnits.flatMap((workUnit) => workUnit.actions);
return {
wikiCount: actions.filter((action) => action.target === 'wiki').length,
slCount: actions.filter((action) => action.target === 'sl').length,
};
}
export function fullDemoCredentialStatus(
project: KloLocalProject,
env: NodeJS.ProcessEnv = process.env,
): FullDemoCredentialStatus {
const llm = project.config.llm;
if (llm.provider.backend === 'none') {
return { status: 'unsupported-provider', provider: llm.provider.backend };
}
if (llm.provider.backend === 'anthropic' && !resolveKloConfigReference(llm.provider.anthropic?.api_key, env)) {
return { status: 'missing-anthropic-key' };
}
return { status: 'ready' };
}
export function assertFullDemoCredentials(project: KloLocalProject, env: NodeJS.ProcessEnv = process.env): void {
const llm = project.config.llm;
const status = fullDemoCredentialStatus(project, env);
if (status.status === 'ready') {
return;
}
if (status.status === 'unsupported-provider') {
throw new Error(
'klo setup demo --mode full requires llm.provider.backend: anthropic, vertex, or gateway. Run `klo setup demo init --force --no-input` to recreate the demo config, or run `klo setup demo --mode seeded --no-input` without credentials.',
);
}
if (llm.provider.backend === 'anthropic') {
throw new Error(
'klo setup demo --mode full needs ANTHROPIC_API_KEY. Export ANTHROPIC_API_KEY and rerun `klo setup demo --mode full --no-input`, or run `klo setup demo --mode seeded --no-input` without credentials.',
);
}
}
export function buildFullDemoReplay(report: IngestReportSnapshot): MemoryFlowReplayInput {
return ingestReportToMemoryFlowReplay(report, { provenanceRowCount: report.body.provenanceRows.length });
}
function initialFullReplay(projectDir: string): MemoryFlowReplayInput {
return {
runId: DEMO_FULL_JOB_ID,
connectionId: DEMO_CONNECTION_ID,
adapter: DEMO_ADAPTER,
status: 'running',
sourceDir: `${projectDir}/raw-sources/${DEMO_CONNECTION_ID}/${DEMO_ADAPTER}`,
syncId: 'pending',
errors: [],
events: [],
plannedWorkUnits: [],
details: { actions: [], provenance: [], transcripts: [] },
};
}
export async function runDemoFull(options: DemoFullOptions): Promise<DemoFullResult> {
await ensureDemoProjectForReuse(options.projectDir);
const project = await loadKloProject({ projectDir: options.projectDir });
assertFullDemoCredentials(project, options.env);
const { result: scan } = await runDemoScan({
projectDir: project.projectDir,
jobId: 'demo-full-scan',
...(options.runLocalScan ? { runLocalScan: options.runLocalScan } : {}),
});
const memoryFlow = options.onMemoryFlowChange
? createMemoryFlowLiveBuffer(initialFullReplay(project.projectDir), { onChange: options.onMemoryFlowChange })
: undefined;
const executeLocalIngest = options.runLocalIngest ?? runLocalIngest;
const ingest = await executeLocalIngest({
project,
adapters: createKloCliLocalIngestAdapters(project),
adapter: DEMO_ADAPTER,
connectionId: DEMO_CONNECTION_ID,
trigger: 'manual_resync',
jobId: DEMO_FULL_JOB_ID,
...(memoryFlow ? { memoryFlow } : {}),
} satisfies RunLocalIngestOptions);
return {
project,
scan,
ingest,
report: ingest.report,
replay: buildFullDemoReplay(ingest.report),
};
}
export function formatFullDemoSummary(report: IngestReportSnapshot): string {
const counts = savedCounts(report);
return [
'Full demo ingest: done',
`Report: ${report.id}`,
`Run: ${report.runId}`,
`Job: ${report.jobId}`,
`Sync: ${report.body.syncId}`,
`Saved memory: ${counts.wikiCount} wiki, ${counts.slCount} semantic layer`,
`Provenance rows: ${report.body.provenanceRows.length}`,
'Next: klo setup demo inspect',
' Shows the files, semantic-layer sources, and memory KLO just produced.',
'Next: klo setup demo replay',
' Replays the same visual story without calling the LLM again.',
'',
].join('\n');
}
const ADAPTER_PREFIXES = ['live_database_', 'metabase_', 'looker_', 'lookml_', 'metricflow_', 'notion_', 'historic_sql_', 'dbt_descriptions_'];
function humanizeUnitKeyForReport(unitKey: string): string {
let key = unitKey.replace(/-/g, '_');
for (const prefix of ADAPTER_PREFIXES) {
if (key.startsWith(prefix)) { key = key.slice(prefix.length); break; }
}
return key.replace(/_/g, ' ');
}
export function formatCleanDemoSummary(report: IngestReportSnapshot, projectDir: string): string {
const counts = savedCounts(report);
const workUnits = report.body.workUnits;
const conflictCount = report.body.conflictsResolved.length;
const areasAnalyzed = workUnits.filter((wu) => wu.actions.length > 0).length;
const lines: string[] = ['', '★ KLO finished ingesting your data', ''];
if (areasAnalyzed > 0) {
lines.push(` ✓ Analyzed ${areasAnalyzed} business area${areasAnalyzed === 1 ? '' : 's'}`);
}
if (!report.body.reconciliationSkipped) {
lines.push(` ✓ Reconciled — ${conflictCount > 0 ? `${conflictCount} conflict${conflictCount === 1 ? '' : 's'} resolved` : 'no conflicts'}`);
}
lines.push('');
if (counts.slCount > 0 || counts.wikiCount > 0) {
lines.push(' KLO created:');
if (counts.slCount > 0) lines.push(` 📊 ${counts.slCount} query definition${counts.slCount === 1 ? '' : 's'} — so agents can write accurate SQL for your data`);
if (counts.wikiCount > 0) lines.push(` 📝 ${counts.wikiCount} knowledge page${counts.wikiCount === 1 ? '' : 's'} — so agents understand your business context`);
lines.push('');
}
const memoryFlow = report.body.memoryFlow;
if (memoryFlow) {
for (const detail of memoryFlow.details.actions) {
if (!detail.summary) continue;
const icon = detail.target === 'sl' ? '📊' : '📝';
lines.push(` ${icon} ${detail.summary}`);
}
}
lines.push('');
lines.push(' What to do next:');
lines.push(...formatNextStepLines());
lines.push('');
lines.push(` Your KLO project files are at: ${projectDir}`);
lines.push('');
return lines.join('\n');
}

View file

@ -0,0 +1,127 @@
import { mkdtemp, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { ensureDemoProject } from './demo-assets.js';
import {
chooseDemoProjectForInteractiveRun,
createTestDemoPromptAdapter,
resolveFullCredentialDecision,
} from './demo-interaction.js';
function io(isTTY: boolean) {
return {
stdin: { isTTY },
stdout: { isTTY, write: vi.fn() },
stderr: { write: vi.fn() },
};
}
describe('demo interaction decisions', () => {
let tempDir: string;
beforeEach(async () => {
tempDir = await mkdtemp(join(tmpdir(), 'klo-demo-interaction-'));
});
afterEach(async () => {
await rm(tempDir, { recursive: true, force: true });
});
it('reuses a valid project without prompting in no-input mode', async () => {
await ensureDemoProject({ projectDir: tempDir, force: false });
await expect(
chooseDemoProjectForInteractiveRun({
projectDir: tempDir,
inputMode: 'disabled',
io: io(false),
prompts: createTestDemoPromptAdapter({ choices: [] }),
}),
).resolves.toEqual({ action: 'use', projectDir: tempDir, reset: false });
});
it('fails corrupted projects in no-input mode with reset guidance', async () => {
await ensureDemoProject({ projectDir: tempDir, force: false });
await rm(join(tempDir, 'demo.db'), { force: true });
await expect(
chooseDemoProjectForInteractiveRun({
projectDir: tempDir,
inputMode: 'disabled',
io: io(false),
prompts: createTestDemoPromptAdapter({ choices: [] }),
}),
).rejects.toThrow(
`Demo project is not ready at ${tempDir}: missing demo.db. Run klo setup demo reset --project-dir ${tempDir} --force --no-input`,
);
});
it('lets interactive users reset a corrupted project', async () => {
await ensureDemoProject({ projectDir: tempDir, force: false });
await rm(join(tempDir, 'demo.db'), { force: true });
await expect(
chooseDemoProjectForInteractiveRun({
projectDir: tempDir,
io: io(true),
prompts: createTestDemoPromptAdapter({ choices: ['reset'], confirms: [true] }),
}),
).resolves.toEqual({ action: 'use', projectDir: tempDir, reset: true });
});
it('lets interactive users choose another project directory', async () => {
await ensureDemoProject({ projectDir: tempDir, force: false });
const otherDir = join(tempDir, 'other-demo');
await expect(
chooseDemoProjectForInteractiveRun({
projectDir: tempDir,
io: io(true),
prompts: createTestDemoPromptAdapter({ choices: ['other'], texts: [otherDir] }),
}),
).resolves.toEqual({ action: 'use', projectDir: otherDir, reset: false });
});
it('uses a pasted Anthropic key only for the returned process env', async () => {
// pragma: allowlist secret
const prompts = createTestDemoPromptAdapter({ choices: ['process_key'], passwords: ['sk-ant-process'] });
await expect(
resolveFullCredentialDecision({
needsAnthropicKey: true,
inputMode: 'auto',
io: io(true),
env: {},
prompts,
}),
).resolves.toEqual({
action: 'full',
env: { ANTHROPIC_API_KEY: 'sk-ant-process' }, // pragma: allowlist secret
});
});
it('lets interactive users explicitly choose seeded mode when the key is missing', async () => {
await expect(
resolveFullCredentialDecision({
needsAnthropicKey: true,
inputMode: 'auto',
io: io(true),
env: {},
prompts: createTestDemoPromptAdapter({ choices: ['seeded'] }),
}),
).resolves.toEqual({ action: 'run-mode', mode: 'seeded' });
});
it('does not prompt when input is disabled', async () => {
await expect(
resolveFullCredentialDecision({
needsAnthropicKey: true,
inputMode: 'disabled',
io: io(false),
env: {},
prompts: createTestDemoPromptAdapter({ choices: ['seeded'] }),
}),
).resolves.toEqual({ action: 'full', env: {} });
});
});

View file

@ -0,0 +1,202 @@
import { cancel, confirm, isCancel, password, select, text } from '@clack/prompts';
import type { Option as ClackOption } from '@clack/prompts';
import { resolve } from 'node:path';
import { inspectDemoProjectState } from './demo-assets.js';
import type { KloDemoInputMode } from './demo.js';
import { withMenuOptionsSpacing } from './prompt-navigation.js';
type DemoPromptOption<T extends string> = ClackOption<T>;
export interface DemoPromptAdapter {
select<T extends string>(options: { message: string; options: Array<DemoPromptOption<T>> }): Promise<T>;
confirm(options: { message: string; initialValue?: boolean }): Promise<boolean>;
password(options: { message: string }): Promise<string>;
text(options: { message: string; placeholder?: string }): Promise<string>;
cancel(message: string): void;
}
interface DemoInteractiveIo {
stdin?: { isTTY?: boolean };
stdout: { isTTY?: boolean };
}
type DemoProjectDecision =
| { action: 'use'; projectDir: string; reset: boolean }
| { action: 'cancel' };
type FullCredentialDecision =
| { action: 'full'; env: NodeJS.ProcessEnv }
| { action: 'run-mode'; mode: 'seeded' | 'replay' }
| { action: 'cancel' };
function isInteractive(inputMode: KloDemoInputMode | undefined, io: DemoInteractiveIo): boolean {
return inputMode !== 'disabled' && io.stdin?.isTTY === true && io.stdout.isTTY === true;
}
function cloneEnv(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv {
return { ...env };
}
function ensureNotCancelled<T>(value: T | symbol, prompts: Pick<DemoPromptAdapter, 'cancel'>): T {
if (isCancel(value)) {
prompts.cancel('Demo cancelled.');
throw new Error('Demo cancelled.');
}
return value as T;
}
export function createClackDemoPromptAdapter(): DemoPromptAdapter {
return {
async select<T extends string>(options: { message: string; options: Array<DemoPromptOption<T>> }): Promise<T> {
return ensureNotCancelled(await select(withMenuOptionsSpacing(options)), this);
},
async confirm(options: { message: string; initialValue?: boolean }): Promise<boolean> {
return ensureNotCancelled(await confirm(options), this);
},
async password(options: { message: string }): Promise<string> {
return ensureNotCancelled(await password(options), this);
},
async text(options: { message: string; placeholder?: string }): Promise<string> {
return ensureNotCancelled(await text(options), this);
},
cancel(message: string): void {
cancel(message);
},
};
}
export function createTestDemoPromptAdapter(options: {
choices?: string[];
confirms?: boolean[];
passwords?: string[];
texts?: string[];
}): DemoPromptAdapter {
const choices = [...(options.choices ?? [])];
const confirms = [...(options.confirms ?? [])];
const passwords = [...(options.passwords ?? [])];
const texts = [...(options.texts ?? [])];
return {
async select<T extends string>(): Promise<T> {
return choices.shift() as T;
},
async confirm(): Promise<boolean> {
return confirms.shift() ?? false;
},
async password(): Promise<string> {
return passwords.shift() ?? '';
},
async text(): Promise<string> {
return texts.shift() ?? '';
},
cancel(): void {
return;
},
};
}
export async function chooseDemoProjectForInteractiveRun(options: {
projectDir: string;
inputMode?: KloDemoInputMode;
io: DemoInteractiveIo;
prompts?: DemoPromptAdapter;
}): Promise<DemoProjectDecision> {
const prompts = options.prompts ?? createClackDemoPromptAdapter();
const projectDir = resolve(options.projectDir);
const state = await inspectDemoProjectState(projectDir);
if (!isInteractive(options.inputMode, options.io)) {
if (state.status === 'corrupt') {
throw new Error(
`Demo project is not ready at ${projectDir}: missing ${state.missing.join(', ')}. Run klo setup demo reset --project-dir ${projectDir} --force --no-input`,
);
}
return { action: 'use', projectDir, reset: false };
}
if (state.status === 'missing') {
return { action: 'use', projectDir, reset: false };
}
const choices =
state.status === 'ready'
? [
{ value: 'reuse', label: 'Reuse existing demo project' },
{ value: 'reset', label: 'Reset demo project' },
{ value: 'other', label: 'Choose another directory' },
{ value: 'cancel', label: 'Cancel' },
]
: [
{ value: 'reset', label: 'Reset corrupted demo project', hint: `Missing ${state.missing.join(', ')}` },
{ value: 'other', label: 'Choose another directory' },
{ value: 'cancel', label: 'Cancel' },
];
const choice = await prompts.select({
message: state.status === 'ready' ? `Demo project exists at ${projectDir}` : `Demo project is not ready at ${projectDir}`,
options: choices,
});
if (choice === 'cancel') {
prompts.cancel('Demo cancelled.');
return { action: 'cancel' };
}
if (choice === 'other') {
const nextProjectDir = await prompts.text({
message: 'Demo project directory',
placeholder: projectDir,
});
return { action: 'use', projectDir: resolve(nextProjectDir), reset: false };
}
if (choice === 'reset') {
const confirmed = await prompts.confirm({
message: `Recreate ${projectDir}? Existing demo artifacts under that directory will be removed.`,
initialValue: false,
});
return confirmed ? { action: 'use', projectDir, reset: true } : { action: 'cancel' };
}
return { action: 'use', projectDir, reset: false };
}
export async function resolveFullCredentialDecision(options: {
needsAnthropicKey: boolean;
inputMode?: KloDemoInputMode;
io: DemoInteractiveIo;
env: NodeJS.ProcessEnv;
prompts?: DemoPromptAdapter;
}): Promise<FullCredentialDecision> {
const env = cloneEnv(options.env);
if (!options.needsAnthropicKey || env.ANTHROPIC_API_KEY) {
return { action: 'full', env };
}
if (!isInteractive(options.inputMode, options.io)) {
return { action: 'full', env };
}
const prompts = options.prompts ?? createClackDemoPromptAdapter();
const choice = await prompts.select({
message: 'Anthropic credentials are missing for the full demo',
options: [
{ value: 'process_key', label: 'Enter key for this process only' },
{ value: 'seeded', label: 'Run pre-seeded demo without LLM' },
{ value: 'replay', label: 'Run packaged replay' },
{ value: 'cancel', label: 'Cancel' },
],
});
if (choice === 'cancel') {
prompts.cancel('Demo cancelled.');
return { action: 'cancel' };
}
if (choice === 'seeded' || choice === 'replay') {
return { action: 'run-mode', mode: choice };
}
const key = await prompts.password({ message: 'ANTHROPIC_API_KEY' });
return { action: 'full', env: { ...env, ANTHROPIC_API_KEY: key } };
}

View file

@ -0,0 +1,137 @@
import type { MemoryFlowEvent, MemoryFlowReplayInput } from '@klo/context/ingest/memory-flow';
import { describe, expect, it } from 'vitest';
import {
buildDemoMetrics,
formatCost,
formatDuration,
formatEta,
formatTokens,
formatTokensPerSec,
progressBar,
} from './demo-metrics.js';
function snapshot(events: MemoryFlowEvent[], overrides: Partial<MemoryFlowReplayInput> = {}): MemoryFlowReplayInput {
return {
runId: 'run-1',
connectionId: 'orbit_demo',
adapter: 'live-database',
status: 'running',
sourceDir: null,
syncId: 'sync-1',
errors: [],
events,
plannedWorkUnits: [],
details: { actions: [], provenance: [], transcripts: [] },
...overrides,
};
}
describe('buildDemoMetrics', () => {
it('estimates elapsed, agent steps, tool calls, and cost from event stream', () => {
const start = Date.UTC(2026, 0, 1, 0, 0, 0);
const input = snapshot(
[
{ type: 'source_acquired', adapter: 'live-database', trigger: 'demo_full', fileCount: 5, emittedAt: new Date(start).toISOString() },
{ type: 'work_unit_started', unitKey: 'orders', skills: [], stepBudget: 40, emittedAt: new Date(start + 1000).toISOString() },
{ type: 'work_unit_step', unitKey: 'orders', stepIndex: 6, stepBudget: 40, emittedAt: new Date(start + 6000).toISOString() },
],
{
plannedWorkUnits: [
{ unitKey: 'orders', rawFiles: [], peerFileCount: 0, dependencyCount: 0 },
{ unitKey: 'customers', rawFiles: [], peerFileCount: 0, dependencyCount: 0 },
],
details: {
actions: [],
provenance: [],
transcripts: [{ unitKey: 'orders', path: '/tmp/orders.jsonl', toolCallCount: 3, errorCount: 0, toolNames: ['x'] }],
},
},
);
const metrics = buildDemoMetrics(input, { now: () => start + 10_000 });
expect(metrics.elapsedMs).toBe(10_000);
expect(metrics.agentSteps).toBe(6);
expect(metrics.agentStepBudget).toBe(40);
expect(metrics.toolCalls).toBe(3);
expect(metrics.workUnitsTotal).toBe(2);
expect(metrics.estimatedTokens).toBeGreaterThan(0);
expect(metrics.estimatedCostUsd).toBeGreaterThan(0);
expect(metrics.isCostEstimated).toBe(true);
});
it('returns null ETA before the first work unit completes', () => {
const input = snapshot([{ type: 'source_acquired', adapter: 'live-database', trigger: 'x', fileCount: 1 }]);
const metrics = buildDemoMetrics(input, { now: () => Date.now() });
expect(metrics.etaMs).toBeNull();
});
it('extrapolates ETA from completed/total ratio when at least one unit finishes', () => {
const start = Date.UTC(2026, 0, 1);
const input = snapshot(
[
{ type: 'source_acquired', adapter: 'a', trigger: 't', fileCount: 1, emittedAt: new Date(start).toISOString() },
{ type: 'work_unit_started', unitKey: 'a', skills: [], stepBudget: 10, emittedAt: new Date(start + 1000).toISOString() },
{ type: 'work_unit_finished', unitKey: 'a', status: 'success', emittedAt: new Date(start + 5000).toISOString() },
],
{
plannedWorkUnits: [
{ unitKey: 'a', rawFiles: [], peerFileCount: 0, dependencyCount: 0 },
{ unitKey: 'b', rawFiles: [], peerFileCount: 0, dependencyCount: 0 },
{ unitKey: 'c', rawFiles: [], peerFileCount: 0, dependencyCount: 0 },
],
},
);
const metrics = buildDemoMetrics(input, { now: () => start + 6_000 });
expect(metrics.etaMs).toBe(12_000);
});
it('reports ETA=0 when the run is finished', () => {
const input = snapshot([], { status: 'done' });
const metrics = buildDemoMetrics(input, { now: () => Date.now() });
expect(metrics.etaMs).toBe(0);
});
});
describe('format helpers', () => {
it('formats duration in s/m/h cascades', () => {
expect(formatDuration(5_000)).toBe('5s');
expect(formatDuration(95_000)).toBe('1m35s');
expect(formatDuration(3_700_000)).toBe('1h01m');
expect(formatDuration(-1)).toBe('--');
});
it('formats ETA as estimating before any data and as duration once running', () => {
expect(formatEta(null, 'running')).toBe('estimating...');
expect(formatEta(8_000, 'running')).toBe('8s');
expect(formatEta(8_000, 'done')).toBe('done');
});
it('formats cost with sub-cent guard', () => {
expect(formatCost(0)).toBe('$0.000');
expect(formatCost(0.0005)).toBe('<$0.001');
expect(formatCost(0.012)).toBe('$0.012');
expect(formatCost(2.5)).toBe('$2.50');
});
it('formats token counts with K/M abbreviations', () => {
expect(formatTokens(0)).toBe('0');
expect(formatTokens(450)).toBe('450');
expect(formatTokens(2_300)).toBe('2.3K');
expect(formatTokens(1_500_000)).toBe('1.50M');
});
it('formats tokens per second', () => {
expect(formatTokensPerSec(0)).toBe('0/s');
expect(formatTokensPerSec(450)).toBe('450/s');
expect(formatTokensPerSec(2300)).toBe('2.3K/s');
});
it('renders a deterministic progress bar with hash and dash characters', () => {
expect(progressBar(0, 10)).toBe('----------');
expect(progressBar(0.5, 10)).toBe('#####-----');
expect(progressBar(1, 10)).toBe('##########');
expect(progressBar(1.4, 10)).toBe('##########');
});
});

View file

@ -0,0 +1,174 @@
import type { MemoryFlowEvent, MemoryFlowReplayInput } from '@klo/context/ingest/memory-flow';
const DEFAULT_INPUT_TOKENS_PER_STEP = 4500;
const DEFAULT_OUTPUT_TOKENS_PER_STEP = 700;
const DEFAULT_INPUT_PRICE_PER_MTOK_USD = 3;
const DEFAULT_OUTPUT_PRICE_PER_MTOK_USD = 15;
interface DemoMetricsTuning {
inputTokensPerStep?: number;
outputTokensPerStep?: number;
inputPricePerMTokUsd?: number;
outputPricePerMTokUsd?: number;
}
interface DemoMetricsSnapshot {
elapsedMs: number;
etaMs: number | null;
agentSteps: number;
agentStepBudget: number;
toolCalls: number;
workUnitsStarted: number;
workUnitsFinished: number;
workUnitsTotal: number;
estimatedInputTokens: number;
estimatedOutputTokens: number;
estimatedTokens: number;
estimatedCostUsd: number;
tokensPerSec: number;
status: MemoryFlowReplayInput['status'];
isCostEstimated: boolean;
}
function eventsOf<T extends MemoryFlowEvent['type']>(
events: MemoryFlowEvent[],
type: T,
): Array<Extract<MemoryFlowEvent, { type: T }>> {
return events.filter((event): event is Extract<MemoryFlowEvent, { type: T }> => event.type === type);
}
function maxAgentStep(events: MemoryFlowEvent[]): { step: number; budget: number } {
const steps = eventsOf(events, 'work_unit_step');
const started = eventsOf(events, 'work_unit_started');
const stepIndex = steps.reduce((max, event) => Math.max(max, event.stepIndex), 0);
const stepBudget = Math.max(
0,
...steps.map((event) => event.stepBudget),
...started.map((event) => event.stepBudget),
);
return { step: stepIndex, budget: stepBudget };
}
function totalToolCalls(input: MemoryFlowReplayInput): number {
return input.details.transcripts.reduce((total, transcript) => total + transcript.toolCallCount, 0);
}
function workUnitProgress(input: MemoryFlowReplayInput): { started: number; finished: number; total: number } {
const started = eventsOf(input.events, 'work_unit_started').length;
const finished = eventsOf(input.events, 'work_unit_finished').length;
const planned = input.plannedWorkUnits.length;
const planEvent = eventsOf(input.events, 'chunks_planned').at(-1);
const total = planned || planEvent?.workUnitCount || started || finished || 0;
return { started, finished, total };
}
function elapsedMsFromEvents(events: MemoryFlowEvent[], nowMs: number): number {
const stamped = events
.map((event) => (event.emittedAt ? Date.parse(event.emittedAt) : Number.NaN))
.filter((value) => Number.isFinite(value));
if (stamped.length === 0) return 0;
const first = Math.min(...stamped);
return Math.max(0, nowMs - first);
}
function estimateEtaMs(
elapsedMs: number,
finished: number,
total: number,
status: MemoryFlowReplayInput['status'],
): number | null {
if (status !== 'running') return 0;
if (total === 0 || finished === 0 || elapsedMs === 0) return null;
const perUnit = elapsedMs / finished;
const remaining = Math.max(0, total - finished);
return Math.round(perUnit * remaining);
}
export function buildDemoMetrics(
input: MemoryFlowReplayInput,
options: { now?: () => number; tuning?: DemoMetricsTuning } = {},
): DemoMetricsSnapshot {
const tuning = options.tuning ?? {};
const inputTokensPerStep = tuning.inputTokensPerStep ?? DEFAULT_INPUT_TOKENS_PER_STEP;
const outputTokensPerStep = tuning.outputTokensPerStep ?? DEFAULT_OUTPUT_TOKENS_PER_STEP;
const inputPrice = tuning.inputPricePerMTokUsd ?? DEFAULT_INPUT_PRICE_PER_MTOK_USD;
const outputPrice = tuning.outputPricePerMTokUsd ?? DEFAULT_OUTPUT_PRICE_PER_MTOK_USD;
const nowMs = (options.now ?? Date.now)();
const elapsedMs = elapsedMsFromEvents(input.events, nowMs);
const { step, budget } = maxAgentStep(input.events);
const toolCalls = totalToolCalls(input);
const progress = workUnitProgress(input);
const finishedCount = eventsOf(input.events, 'work_unit_finished').length;
const stepDriver = Math.max(step, toolCalls, finishedCount * 4);
const inputTokens = stepDriver * inputTokensPerStep;
const outputTokens = stepDriver * outputTokensPerStep;
const totalTokens = inputTokens + outputTokens;
const cost = (inputTokens / 1_000_000) * inputPrice + (outputTokens / 1_000_000) * outputPrice;
const elapsedSec = elapsedMs / 1000;
const tokensPerSec = elapsedSec > 0 ? totalTokens / elapsedSec : 0;
return {
elapsedMs,
etaMs: estimateEtaMs(elapsedMs, progress.finished, progress.total, input.status),
agentSteps: step,
agentStepBudget: budget,
toolCalls,
workUnitsStarted: progress.started,
workUnitsFinished: progress.finished,
workUnitsTotal: progress.total,
estimatedInputTokens: inputTokens,
estimatedOutputTokens: outputTokens,
estimatedTokens: totalTokens,
estimatedCostUsd: cost,
tokensPerSec,
status: input.status,
isCostEstimated: true,
};
}
export function formatDuration(ms: number): string {
if (!Number.isFinite(ms) || ms < 0) return '--';
const totalSec = Math.round(ms / 1000);
if (totalSec < 60) return `${totalSec}s`;
const min = Math.floor(totalSec / 60);
const sec = totalSec % 60;
if (min < 60) return `${min}m${sec.toString().padStart(2, '0')}s`;
const hr = Math.floor(min / 60);
return `${hr}h${(min % 60).toString().padStart(2, '0')}m`;
}
export function formatEta(ms: number | null, status: MemoryFlowReplayInput['status']): string {
if (status !== 'running') return 'done';
if (ms === null) return 'estimating...';
return formatDuration(ms);
}
export function formatCost(usd: number): string {
if (!Number.isFinite(usd) || usd <= 0) return '$0.000';
if (usd < 0.001) return '<$0.001';
if (usd < 1) return `$${usd.toFixed(3)}`;
return `$${usd.toFixed(2)}`;
}
export function formatTokens(n: number): string {
if (!Number.isFinite(n) || n <= 0) return '0';
if (n < 1000) return `${Math.round(n)}`;
if (n < 1_000_000) return `${(n / 1000).toFixed(1)}K`;
return `${(n / 1_000_000).toFixed(2)}M`;
}
export function formatTokensPerSec(n: number): string {
if (!Number.isFinite(n) || n <= 0) return '0/s';
if (n < 1000) return `${Math.round(n)}/s`;
return `${(n / 1000).toFixed(1)}K/s`;
}
const PROGRESS_BAR_WIDTH = 12;
export function progressBar(ratio: number, width: number = PROGRESS_BAR_WIDTH): string {
const clamped = Math.max(0, Math.min(1, ratio));
const filled = Math.round(clamped * width);
return `${'#'.repeat(filled)}${'-'.repeat(Math.max(0, width - filled))}`;
}

View file

@ -0,0 +1,228 @@
import type { MemoryFlowEvent, MemoryFlowReplayInput } from '@klo/context/ingest/memory-flow';
import { describe, expect, it } from 'vitest';
import { createPlainProgressEmitter, formatMemoryFlowEventLine } from './demo-progress.js';
function snapshot(events: MemoryFlowEvent[]): MemoryFlowReplayInput {
return {
runId: 'run-1',
connectionId: 'orbit_demo',
adapter: 'live-database',
status: 'running',
sourceDir: null,
syncId: 'sync-1',
errors: [],
events,
plannedWorkUnits: [],
details: { actions: [], provenance: [], transcripts: [] },
};
}
describe('formatMemoryFlowEventLine', () => {
it('formats source_acquired in plain English with adapter and file count', () => {
expect(
formatMemoryFlowEventLine({
type: 'source_acquired',
adapter: 'live-database',
trigger: 'manual_resync',
fileCount: 7,
}),
).toBe('[connect] Connected live-database - 7 database files (manual_resync)');
});
it('formats diff_computed as a comma-separated breakdown', () => {
expect(
formatMemoryFlowEventLine({
type: 'diff_computed',
added: 3,
modified: 1,
deleted: 0,
unchanged: 4,
}),
).toBe('[diff] Tables: +3 new, ~1 changed, =4 unchanged');
});
it('formats diff_computed as "no changes" when every counter is zero', () => {
expect(
formatMemoryFlowEventLine({
type: 'diff_computed',
added: 0,
modified: 0,
deleted: 0,
unchanged: 0,
}),
).toBe('[diff] Tables: no changes');
});
it('formats chunks_planned without removals as a single readable sentence', () => {
expect(
formatMemoryFlowEventLine({
type: 'chunks_planned',
chunkCount: 7,
workUnitCount: 5,
evictionCount: 0,
}),
).toBe('[plan] Grouped 5 tables into 7 business areas');
});
it('formats chunks_planned with removals when evictions are non-zero', () => {
expect(
formatMemoryFlowEventLine({
type: 'chunks_planned',
chunkCount: 7,
workUnitCount: 5,
evictionCount: 2,
}),
).toBe('[plan] Grouped 5 tables into 7 business areas (2 removals)');
});
it('formats work_unit_started in human terms', () => {
expect(
formatMemoryFlowEventLine({
type: 'work_unit_started',
unitKey: 'revenue-policy',
skills: ['sl_expert', 'wiki_writer'],
stepBudget: 40,
}),
).toBe('[analyze] Reviewing "revenue-policy" - budget 40 agent steps');
});
it('suppresses noisy work_unit_step events', () => {
expect(
formatMemoryFlowEventLine({
type: 'work_unit_step',
unitKey: 'revenue-policy',
stepIndex: 3,
stepBudget: 40,
}),
).toBeNull();
});
it('formats candidate_action with friendly target and arrow', () => {
expect(
formatMemoryFlowEventLine({
type: 'candidate_action',
unitKey: 'revenue-policy',
target: 'sl',
action: 'created',
key: 'warehouse.revenue',
}),
).toBe('[draft] revenue-policy -> semantic-layer: created warehouse.revenue');
});
it('formats work_unit_finished with status-aware tag', () => {
expect(
formatMemoryFlowEventLine({
type: 'work_unit_finished',
unitKey: 'revenue-policy',
status: 'success',
}),
).toBe('[done] revenue-policy reviewed');
expect(
formatMemoryFlowEventLine({
type: 'work_unit_finished',
unitKey: 'revenue-policy',
status: 'failed',
reason: 'budget exhausted',
}),
).toBe('[fail] revenue-policy needs attention - budget exhausted');
});
it('formats reconciliation_finished with friendly counter wording', () => {
expect(
formatMemoryFlowEventLine({
type: 'reconciliation_finished',
conflictCount: 0,
fallbackCount: 0,
}),
).toBe('[validate] Reconciled drafts - no conflicts, nothing flagged for review');
expect(
formatMemoryFlowEventLine({
type: 'reconciliation_finished',
conflictCount: 2,
fallbackCount: 1,
}),
).toBe('[validate] Reconciled drafts - 2 conflicts, 1 item flagged for review');
});
it('formats saved with optional shortened commit sha and pluralized memory count', () => {
expect(
formatMemoryFlowEventLine({
type: 'saved',
commitSha: 'abc1234567890', // pragma: allowlist secret
wikiCount: 2,
slCount: 5,
}),
).toBe('[memory] Saved 7 memories (2 wiki, 5 semantic-layer) - commit abc1234');
expect(
formatMemoryFlowEventLine({
type: 'saved',
commitSha: null,
wikiCount: 0,
slCount: 1,
}),
).toBe('[memory] Saved 1 memory (0 wiki, 1 semantic-layer)');
});
it('formats report_created with run id', () => {
expect(
formatMemoryFlowEventLine({
type: 'report_created',
runId: 'run-xyz',
}),
).toBe('[report] Run report ready: run-xyz');
});
});
describe('createPlainProgressEmitter', () => {
it('writes one line per new event and never re-emits prior events', () => {
const written: string[] = [];
const io = {
stdout: { write: (chunk: string) => written.push(chunk), isTTY: false },
stderr: { write: () => undefined },
};
const emit = createPlainProgressEmitter(io);
emit(
snapshot([
{ type: 'source_acquired', adapter: 'live-database', trigger: 'manual_resync', fileCount: 7 },
{ type: 'diff_computed', added: 0, modified: 0, deleted: 0, unchanged: 7 },
]),
);
emit(
snapshot([
{ type: 'source_acquired', adapter: 'live-database', trigger: 'manual_resync', fileCount: 7 },
{ type: 'diff_computed', added: 0, modified: 0, deleted: 0, unchanged: 7 },
{ type: 'work_unit_started', unitKey: 'revenue-policy', skills: ['sl_expert'], stepBudget: 40 },
]),
);
expect(written).toEqual([
'[connect] Connected live-database - 7 database files (manual_resync)\n',
'[diff] Tables: =7 unchanged\n',
'[analyze] Reviewing "revenue-policy" - budget 40 agent steps\n',
]);
});
it('skips suppressed events without advancing visible output', () => {
const written: string[] = [];
const io = {
stdout: { write: (chunk: string) => written.push(chunk), isTTY: false },
stderr: { write: () => undefined },
};
const emit = createPlainProgressEmitter(io);
emit(
snapshot([
{ type: 'work_unit_step', unitKey: 'a', stepIndex: 1, stepBudget: 40 },
{ type: 'work_unit_step', unitKey: 'a', stepIndex: 2, stepBudget: 40 },
{ type: 'work_unit_finished', unitKey: 'a', status: 'success' },
]),
);
expect(written).toEqual(['[done] a reviewed\n']);
});
});

View file

@ -0,0 +1,77 @@
import type { MemoryFlowEvent, MemoryFlowReplayInput } from '@klo/context/ingest/memory-flow';
import type { KloDemoIo } from './demo.js';
function plural(n: number, one: string, many = `${one}s`): string {
return `${n} ${n === 1 ? one : many}`;
}
function formatDiff(added: number, modified: number, deleted: number, unchanged: number): string {
const parts: string[] = [];
if (added > 0) parts.push(`+${added} new`);
if (modified > 0) parts.push(`~${modified} changed`);
if (deleted > 0) parts.push(`-${deleted} removed`);
if (unchanged > 0) parts.push(`=${unchanged} unchanged`);
return parts.length > 0 ? parts.join(', ') : 'no changes';
}
export function formatMemoryFlowEventLine(event: MemoryFlowEvent): string | null {
switch (event.type) {
case 'source_acquired':
return `[connect] Connected ${event.adapter} - ${plural(event.fileCount, 'database file')} (${event.trigger})`;
case 'scope_detected':
return event.fingerprint
? `[scope] Scope locked: ${event.fingerprint}`
: '[scope] Reviewing the whole warehouse (no scope filter)';
case 'raw_snapshot_written':
return `[snapshot] Captured snapshot ${event.syncId} - ${plural(event.rawFileCount, 'file')}`;
case 'diff_computed':
return `[diff] Tables: ${formatDiff(event.added, event.modified, event.deleted, event.unchanged)}`;
case 'chunks_planned':
return event.evictionCount > 0
? `[plan] Grouped ${plural(event.workUnitCount, 'table')} into ${plural(event.chunkCount, 'business area')} (${plural(event.evictionCount, 'removal')})`
: `[plan] Grouped ${plural(event.workUnitCount, 'table')} into ${plural(event.chunkCount, 'business area')}`;
case 'stage_skipped':
return `[skip] ${event.stage} skipped: ${event.reason}`;
case 'work_unit_started':
return `[analyze] Reviewing "${event.unitKey}" - budget ${plural(event.stepBudget, 'agent step')}`;
case 'work_unit_step':
return null;
case 'candidate_action': {
const target = event.target === 'sl' ? 'semantic-layer' : 'wiki';
return `[draft] ${event.unitKey} -> ${target}: ${event.action} ${event.key}`;
}
case 'work_unit_finished':
if (event.status === 'success') {
return `[done] ${event.unitKey} reviewed`;
}
return `[fail] ${event.unitKey} needs attention${event.reason ? ` - ${event.reason}` : ''}`;
case 'reconciliation_finished': {
const conflicts = event.conflictCount === 0 ? 'no conflicts' : plural(event.conflictCount, 'conflict');
const fallbacks = event.fallbackCount === 0 ? 'nothing flagged for review' : `${plural(event.fallbackCount, 'item')} flagged for review`;
return `[validate] Reconciled drafts - ${conflicts}, ${fallbacks}`;
}
case 'saved': {
const total = event.wikiCount + event.slCount;
const commit = event.commitSha ? ` - commit ${event.commitSha.slice(0, 7)}` : '';
return `[memory] Saved ${plural(total, 'memory', 'memories')} (${event.wikiCount} wiki, ${event.slCount} semantic-layer)${commit}`;
}
case 'provenance_recorded':
return `[trace] Recorded provenance for ${plural(event.rowCount, 'row')}`;
case 'report_created':
return `[report] Run report ready: ${event.runId}`;
}
}
export function createPlainProgressEmitter(io: KloDemoIo): (snapshot: MemoryFlowReplayInput) => void {
let printed = 0;
return (snapshot) => {
while (printed < snapshot.events.length) {
const event = snapshot.events[printed++];
if (!event) continue;
const line = formatMemoryFlowEventLine(event);
if (line !== null) {
io.stdout.write(`${line}\n`);
}
}
};
}

View file

@ -0,0 +1,60 @@
import { mkdtemp, readFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { type MemoryFlowReplayInput } from '@klo/context/ingest/memory-flow';
import { describe, expect, it } from 'vitest';
import { DEMO_LATEST_REPLAY_FILE, loadLatestDemoReplay, writeDemoReplay } from './demo-replay-store.js';
function replay(overrides: Partial<MemoryFlowReplayInput> = {}): MemoryFlowReplayInput {
return {
metadata: {
schemaVersion: 1,
mode: 'full',
origin: 'captured',
timing: 'captured',
capturedAt: '2026-05-01T10:00:03.000Z',
sourceReportId: 'report-1',
sourceReportPath: 'report-1',
fallbackReason: null,
},
runId: 'run-1',
connectionId: 'orbit_demo',
adapter: 'live-database',
status: 'done',
sourceDir: null,
syncId: 'sync-1',
reportId: 'report-1',
reportPath: 'report-1',
errors: [],
events: [{ type: 'report_created', runId: 'run-1', reportPath: 'report-1', emittedAt: '2026-05-01T10:00:03.000Z' }],
plannedWorkUnits: [],
details: { actions: [], provenance: [], transcripts: [] },
...overrides,
};
}
describe('demo replay store', () => {
it('writes a versioned replay file and updates latest', async () => {
const projectDir = await mkdtemp(join(tmpdir(), 'klo-demo-replay-store-'));
const saved = await writeDemoReplay(projectDir, replay(), { label: 'full' });
expect(saved.replayPath).toMatch(/replays[/\\]full-run-1.memory-flow.v1.json$/);
expect(saved.latestReplayPath).toBe(join(projectDir, 'replays', DEMO_LATEST_REPLAY_FILE));
expect(await loadLatestDemoReplay(projectDir)).toMatchObject({
runId: 'run-1',
metadata: { mode: 'full', origin: 'captured', timing: 'captured' },
});
const wrapper = JSON.parse(await readFile(saved.latestReplayPath, 'utf-8')) as {
memoryFlowReplaySchemaVersion?: number;
};
expect(wrapper.memoryFlowReplaySchemaVersion).toBe(1);
});
it('returns null when no latest local replay exists', async () => {
const projectDir = await mkdtemp(join(tmpdir(), 'klo-demo-replay-store-empty-'));
await expect(loadLatestDemoReplay(projectDir)).resolves.toBeNull();
});
});

View file

@ -0,0 +1,68 @@
import { constants as fsConstants } from 'node:fs';
import { access, copyFile, mkdir, readFile, writeFile } from 'node:fs/promises';
import { join, resolve } from 'node:path';
import { parseMemoryFlowReplayInput, type MemoryFlowReplayInput } from '@klo/context/ingest/memory-flow';
interface StoredMemoryFlowReplayFile {
memoryFlowReplaySchemaVersion: 1;
replay: unknown;
}
interface SavedDemoReplay {
replayPath: string;
latestReplayPath: string;
}
export const DEMO_LATEST_REPLAY_FILE = 'latest.memory-flow.v1.json';
async function exists(path: string): Promise<boolean> {
try {
await access(path, fsConstants.F_OK);
return true;
} catch {
return false;
}
}
function safeReplayName(value: string): string {
return value.replace(/[^a-zA-Z0-9._-]+/g, '-').replace(/^-+|-+$/g, '') || 'replay';
}
function demoReplayFileName(input: MemoryFlowReplayInput, label: string): string {
return `${safeReplayName(label)}-${safeReplayName(input.runId)}.memory-flow.v1.json`;
}
function wrapReplay(input: MemoryFlowReplayInput): StoredMemoryFlowReplayFile {
return { memoryFlowReplaySchemaVersion: 1, replay: input };
}
export async function loadDemoReplayFile(path: string): Promise<MemoryFlowReplayInput> {
const parsed = JSON.parse(await readFile(path, 'utf-8')) as StoredMemoryFlowReplayFile;
if (parsed.memoryFlowReplaySchemaVersion !== 1) {
throw new Error(`Unsupported demo replay schema version in ${path}`);
}
return parseMemoryFlowReplayInput(parsed.replay);
}
export async function loadLatestDemoReplay(projectDir: string): Promise<MemoryFlowReplayInput | null> {
const latestPath = join(resolve(projectDir), 'replays', DEMO_LATEST_REPLAY_FILE);
if (!(await exists(latestPath))) {
return null;
}
return loadDemoReplayFile(latestPath);
}
export async function writeDemoReplay(
projectDir: string,
input: MemoryFlowReplayInput,
options: { label: 'full' | 'deterministic' | 'seeded' },
): Promise<SavedDemoReplay> {
const replayDir = join(resolve(projectDir), 'replays');
await mkdir(replayDir, { recursive: true });
const replayPath = join(replayDir, demoReplayFileName(input, options.label));
const latestReplayPath = join(replayDir, DEMO_LATEST_REPLAY_FILE);
const body = `${JSON.stringify(wrapReplay(input), null, 2)}\n`;
await writeFile(replayPath, body, 'utf-8');
await copyFile(replayPath, latestReplayPath);
return { replayPath, latestReplayPath };
}

View file

@ -0,0 +1,31 @@
import { rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { afterEach, describe, expect, it } from 'vitest';
import { findLatestDemoScanReport, runDemoScan } from './demo-scan.js';
describe('demo scan helpers', () => {
const projectDir = join(tmpdir(), `klo-demo-scan-${process.pid}`);
afterEach(async () => {
await rm(projectDir, { recursive: true, force: true });
});
it('runs the packaged SQLite demo scan and finds the latest scan report', async () => {
const { result } = await runDemoScan({
projectDir,
jobId: 'demo-scan-test',
now: () => new Date('2026-05-06T10:00:00.000Z'),
});
expect(result.report).toMatchObject({
connectionId: 'orbit_demo',
driver: 'sqlite',
runId: 'demo-scan-test',
mode: 'structural',
dryRun: false,
});
expect(result.report.artifactPaths.reportPath).toContain('raw-sources/orbit_demo/live-database/');
await expect(findLatestDemoScanReport(projectDir)).resolves.toMatchObject({ runId: 'demo-scan-test' });
});
});

View file

@ -0,0 +1,223 @@
import { getLocalIngestStatus, type IngestReportSnapshot, type MemoryFlowReplayInput } from '@klo/context/ingest';
import { loadKloProject, type KloLocalProject } from '@klo/context/project';
import { runLocalScan, type KloScanReport, type LocalScanRunResult } from '@klo/context/scan';
import { DEMO_ADAPTER, DEMO_CONNECTION_ID, DEMO_FULL_JOB_ID, ensureDemoProject } from './demo-assets.js';
import { loadLatestDemoReplay } from './demo-replay-store.js';
import { createKloCliLocalIngestAdapters } from './local-adapters.js';
interface DemoScanOptions {
projectDir: string;
jobId?: string;
now?: () => Date;
runLocalScan?: typeof runLocalScan;
}
interface DemoScanResult {
project: KloLocalProject;
result: LocalScanRunResult;
}
interface DemoInspectSummary {
projectDir: string;
scanReport: KloScanReport | null;
fullReport: IngestReportSnapshot | null;
semanticLayerFileCount: number;
knowledgeFileCount: number;
replayFileCount: number;
latestReplay: MemoryFlowReplayInput | null;
}
interface DemoInspectDeps {
findFullReport?: (project: KloLocalProject) => Promise<IngestReportSnapshot | null>;
}
async function ensureDemoProjectForReuse(projectDir: string): Promise<void> {
await ensureDemoProject({ projectDir, force: false }).catch((error) => {
if (error instanceof Error && error.message.includes('Demo project already exists')) {
return;
}
throw error;
});
}
async function loadReadyDemoProject(projectDir: string): Promise<KloLocalProject> {
try {
return await loadKloProject({ projectDir });
} catch (error) {
const reason = error instanceof Error ? error.message : String(error);
throw new Error(
`Demo project is not ready at ${projectDir}: ${reason}. Run klo setup demo init --project-dir ${projectDir} --force --no-input to recreate it.`,
);
}
}
function reportDiff(report: KloScanReport): string {
return `+${report.diffSummary.tablesAdded}/~${report.diffSummary.tablesModified}/-${report.diffSummary.tablesDeleted}/=${report.diffSummary.tablesUnchanged}`;
}
function jsonReport(raw: string, path: string): KloScanReport {
try {
return JSON.parse(raw) as KloScanReport;
} catch (error) {
const reason = error instanceof Error ? error.message : String(error);
throw new Error(`Invalid demo scan report at ${path}: ${reason}`);
}
}
async function countFiles(project: KloLocalProject, root: string, predicate: (path: string) => boolean): Promise<number> {
const { files } = await project.fileStore.listFiles(root, true);
return files.filter(predicate).length;
}
async function findFullDemoReport(project: KloLocalProject): Promise<IngestReportSnapshot | null> {
return getLocalIngestStatus(project, DEMO_FULL_JOB_ID);
}
function savedCounts(report: IngestReportSnapshot): { wikiCount: number; slCount: number } {
const actions = report.body.workUnits.flatMap((workUnit) => workUnit.actions);
return {
wikiCount: actions.filter((action) => action.target === 'wiki').length,
slCount: actions.filter((action) => action.target === 'sl').length,
};
}
export async function runDemoScan(options: DemoScanOptions): Promise<DemoScanResult> {
await ensureDemoProjectForReuse(options.projectDir);
const project = await loadReadyDemoProject(options.projectDir);
const executeScan = options.runLocalScan ?? runLocalScan;
const result = await executeScan({
project,
connectionId: DEMO_CONNECTION_ID,
mode: 'structural',
trigger: 'cli',
jobId: options.jobId ?? 'demo-scan',
now: options.now,
adapters: createKloCliLocalIngestAdapters(project),
});
return { project, result };
}
export async function findLatestDemoScanReport(projectDir: string): Promise<KloScanReport | null> {
const project = await loadReadyDemoProject(projectDir);
const root = `raw-sources/${DEMO_CONNECTION_ID}/${DEMO_ADAPTER}`;
const { files } = await project.fileStore.listFiles(root, true);
const latest = files
.filter((path) => path.endsWith('/scan-report.json'))
.sort()
.at(-1);
if (!latest) {
return null;
}
const reportPath = `${root}/${latest}`;
const report = await project.fileStore.readFile(reportPath);
return jsonReport(report.content, reportPath);
}
export async function inspectDemoProject(
projectDir: string,
projectOverride?: KloLocalProject,
deps: DemoInspectDeps = {},
): Promise<DemoInspectSummary> {
const project = projectOverride ?? (await loadReadyDemoProject(projectDir));
const scanReport = await findLatestDemoScanReport(project.projectDir);
const fullReport = await (deps.findFullReport ?? findFullDemoReport)(project);
const semanticLayerFileCount = await countFiles(
project,
`semantic-layer/${DEMO_CONNECTION_ID}`,
(path) => path.endsWith('.yaml') || path.endsWith('.yml'),
);
const knowledgeFileCount = await countFiles(project, 'knowledge', (path) => path.endsWith('.md'));
const replayFileCount = await countFiles(project, 'replays', (path) => path.endsWith('.json'));
const latestReplay = await loadLatestDemoReplay(project.projectDir);
return {
projectDir: project.projectDir,
scanReport,
fullReport,
semanticLayerFileCount,
knowledgeFileCount,
replayFileCount,
latestReplay,
};
}
export function formatDemoScanSummary(report: KloScanReport): string {
return [
'Demo scan: done',
`Connection: ${report.connectionId}`,
`Driver: ${report.driver}`,
`Mode: ${report.mode}`,
`Tables: ${reportDiff(report)}`,
`Semantic-layer artifacts: ${report.artifactPaths.manifestShards.length}`,
`Report: ${report.artifactPaths.reportPath ?? 'none'}`,
'Next: klo setup demo inspect',
' Shows the files and semantic-layer draft created from the database scan.',
'',
].join('\n');
}
function replayLine(replay: MemoryFlowReplayInput | null): string {
if (!replay?.metadata) {
return 'Latest replay: packaged demo replay';
}
return `Latest replay: ${replay.metadata.mode} (${replay.metadata.origin}, ${replay.metadata.timing})`;
}
export function formatDemoInspect(summary: DemoInspectSummary): string {
const report = summary.scanReport;
const fullReport = summary.fullReport;
const fullCounts = fullReport ? savedCounts(fullReport) : null;
const scanLines = report
? [
'Scan artifacts: yes',
`Connection: ${report.connectionId}`,
`Driver: ${report.driver}`,
`Tables: ${reportDiff(report)}`,
`Report: ${report.artifactPaths.reportPath ?? 'none'}`,
]
: ['Scan artifacts: none'];
const memoryLines = fullReport
? [
'Memory synthesis: ran',
`Full report: ${fullReport.id}`,
`Full run: ${fullReport.runId}`,
`Saved memory: ${fullCounts?.wikiCount ?? 0} wiki, ${fullCounts?.slCount ?? 0} semantic layer`,
`Provenance rows: ${fullReport.body.provenanceRows.length}`,
]
: [report ? 'Memory synthesis: full mode not run' : 'Memory synthesis: not run'];
const next = fullReport
? [
`Next: klo ingest watch ${fullReport.runId} --project-dir ${summary.projectDir}`,
' Opens the captured run timeline and lets you inspect what happened.',
'Next: klo setup demo replay',
' Replays the same visual story without calling the LLM again.',
]
: report
? [
'Next: klo setup demo --mode full',
' Runs the full AI-backed pass with your LLM provider.',
'Next: klo setup demo replay',
' Replays the packaged visual story without calling the LLM.',
]
: [
'Next: klo setup demo --no-input',
' Runs the pre-seeded demo without calling the LLM.',
'Next: klo setup demo --mode full',
' Runs the full AI-backed pass with your LLM provider.',
];
return [
`Demo project: ${summary.projectDir}`,
...scanLines,
`Semantic-layer files: ${summary.semanticLayerFileCount}`,
`Knowledge files: ${summary.knowledgeFileCount}`,
`Replay files: ${summary.replayFileCount}`,
replayLine(summary.latestReplay),
...memoryLines,
...next,
'',
].join('\n');
}

View file

@ -0,0 +1,123 @@
import { access, readFile, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { afterEach, describe, expect, it } from 'vitest';
import { runDemoSeeded } from './demo-seeded.js';
import { formatSeededInspect, inspectSeededProject } from './demo-seeded-inspect.js';
import { KLO_NEXT_STEP_COMMANDS } from './next-steps.js';
describe('seeded demo inspect contract', () => {
const projectDir = join(tmpdir(), `klo-demo-seeded-inspect-${process.pid}`);
afterEach(async () => {
await rm(projectDir, { recursive: true, force: true });
});
it('reports the PRD source inventory, generated outputs, status, metadata, and next commands', async () => {
await runDemoSeeded({ projectDir });
const inspect = await inspectSeededProject(projectDir);
expect(inspect).toMatchObject({
projectDir,
mode: 'seeded',
status: { status: 'ready', missing: [] },
modeMetadata: {
mode: 'seeded',
source: 'packaged demo project',
generatedContext: 'prebuilt from bundled assets',
llmCalls: 'none',
origin: 'packaged',
timing: 'prebuilt',
sourceReportId: 'demo-seeded-report',
sourceReportPath: 'reports/seeded-demo-report.json',
},
sourceBundle: {
warehouse: {
label: 'Warehouse',
path: 'demo.db',
tableCount: 8,
totalRows: 11234,
rowCounts: {
accounts: 210,
arr_movements: 720,
contracts: 320,
invoices: 3000,
plans: 4,
purchase_requests: 5200,
support_tickets: 520,
users: 1260,
},
},
dbt: { label: 'dbt', path: 'raw-sources/dbt', modelCount: 3, sourceTableCount: 8 },
bi: { label: 'BI', path: 'raw-sources/bi', exploreCount: 5, dashboardCount: 2 },
notion: { label: 'Notion', path: 'raw-sources/notion', pageCount: 8 },
},
generatedOutputs: {
semanticLayer: { path: 'semantic-layer/orbit_demo', manifestSourceCount: 6, fileCount: 6 },
knowledge: { path: 'knowledge/global', manifestPageCount: 10, fileCount: 10 },
links: { path: 'links/provenance.json', manifestLinkCount: 23, linkCount: 23 },
reports: { primaryPath: 'reports/seeded-demo-report.json', fileCount: 1 },
replays: { primaryPath: 'replays/replay.memory-flow.v1.json', latestPath: 'replays/latest.memory-flow.v1.json' },
},
nextCommands: KLO_NEXT_STEP_COMMANDS,
});
expect(inspect.generatedOutputs.replays.fileCount).toBeGreaterThanOrEqual(3);
await expect(access(join(projectDir, inspect.generatedOutputs.reports.primaryPath))).resolves.toBeUndefined();
await expect(access(join(projectDir, inspect.generatedOutputs.replays.primaryPath))).resolves.toBeUndefined();
await expect(access(join(projectDir, inspect.generatedOutputs.replays.latestPath))).resolves.toBeUndefined();
});
it('formats seeded inspect from the normalized contract', async () => {
await runDemoSeeded({ projectDir });
const output = formatSeededInspect(await inspectSeededProject(projectDir));
expect(output).toContain(`Demo project: ${projectDir}`);
expect(output).toContain('Status: ready');
expect(output).toContain('Mode: seeded (pre-seeded demo project)');
expect(output).toContain('Source: packaged demo project');
expect(output).toContain('Generated context: prebuilt from bundled assets');
expect(output).toContain('LLM calls: none');
expect(output).toContain('Warehouse: 8 tables, 11,234 rows');
expect(output).toContain('Rows: accounts 210, arr_movements 720, contracts 320, invoices 3000');
expect(output).toContain('dbt: 3 models, 8 source tables');
expect(output).toContain('BI: 5 explores, 2 dashboards');
expect(output).toContain('Notion: 8 pages');
expect(output).toContain('Semantic-layer sources: 6 manifest, 6 files');
expect(output).toContain('Knowledge pages: 10 manifest, 10 files');
expect(output).toContain('Evidence links: 23 manifest, 23 links');
expect(output).toContain('Report: reports/seeded-demo-report.json');
expect(output).toContain('Replay: replays/replay.memory-flow.v1.json');
expect(output).toContain('Latest replay: seeded (packaged, prebuilt)');
expect(output).toContain(' $ klo agent tools --json');
expect(output).toContain(' $ klo agent context --json');
expect(output).toContain(' $ klo serve --mcp stdio --user-id local');
expect(output.indexOf('klo agent tools --json')).toBeLessThan(
output.indexOf('klo serve --mcp stdio --user-id local'),
);
expect(output).not.toContain('klo ask');
expect(output).not.toContain('deterministic mode');
});
it('reports missing seeded paths without reading stale counts as ready', async () => {
await runDemoSeeded({ projectDir });
await rm(join(projectDir, 'links', 'provenance.json'));
const inspect = await inspectSeededProject(projectDir);
expect(inspect.status).toEqual({ status: 'corrupt', missing: ['links/provenance.json'] });
expect(formatSeededInspect(inspect)).toContain('Status: corrupt');
expect(formatSeededInspect(inspect)).toContain('Missing: links/provenance.json');
});
it('keeps provenance link counts tied to the project file', async () => {
await runDemoSeeded({ projectDir });
const inspect = await inspectSeededProject(projectDir);
const raw = await readFile(join(projectDir, 'links', 'provenance.json'), 'utf-8');
const links = JSON.parse(raw) as unknown[];
expect(inspect.generatedOutputs.links.linkCount).toBe(links.length);
expect(inspect.generatedOutputs.links.linkCount).toBe(23);
});
});

View file

@ -0,0 +1,299 @@
import { constants as fsConstants } from 'node:fs';
import { access, readFile, readdir } from 'node:fs/promises';
import { join, resolve } from 'node:path';
import type { MemoryFlowReplayInput } from '@klo/context/ingest/memory-flow';
import { loadPackagedDemoReplay } from './demo-assets.js';
import { DEMO_LATEST_REPLAY_FILE, loadLatestDemoReplay } from './demo-replay-store.js';
import { KLO_NEXT_STEP_COMMANDS, KLO_NEXT_STEP_COMMAND_WIDTH } from './next-steps.js';
type SeededInspectReadiness = 'missing' | 'ready' | 'corrupt';
export interface DemoSeededManifest {
demoAssetSchemaVersion: number;
name: string;
displayName: string;
mode: string;
source?: string;
sources: {
warehouse: { label: string; path?: string; tables: number; rowCounts: Record<string, number> };
dbt: { label: string; path?: string; models: number; sourceTables: number };
bi: { label: string; path?: string; explores: number; dashboards: number };
notion: { label: string; path?: string; pages: number };
};
generated: {
semanticLayer: { path?: string; sourceCount: number };
knowledge: { path?: string; pageCount: number };
links: { path?: string; linkCount: number };
};
}
export interface SeededInspectSummary {
projectDir: string;
mode: 'seeded';
manifest: DemoSeededManifest;
status: { status: SeededInspectReadiness; missing: string[] };
sourceBundle: {
warehouse: {
label: string;
path: string;
tableCount: number;
rowCounts: Record<string, number>;
totalRows: number;
};
dbt: { label: string; path: string; modelCount: number; sourceTableCount: number };
bi: { label: string; path: string; exploreCount: number; dashboardCount: number };
notion: { label: string; path: string; pageCount: number };
};
generatedOutputs: {
semanticLayer: { path: string; manifestSourceCount: number; fileCount: number };
knowledge: { path: string; manifestPageCount: number; fileCount: number };
links: { path: string; manifestLinkCount: number; linkCount: number };
reports: { primaryPath: string; fileCount: number };
replays: { primaryPath: string; latestPath: string; fileCount: number };
};
modeMetadata: {
mode: 'seeded';
source: 'packaged demo project';
generatedContext: 'prebuilt from bundled assets';
llmCalls: 'none';
origin: string;
timing: string;
sourceReportId: string | null;
sourceReportPath: string | null;
};
nextCommands: Array<{ command: string; description: string }>;
latestReplay: MemoryFlowReplayInput | null;
}
const REQUIRED_SEEDED_PROJECT_PATHS = [
'klo.yaml',
'demo.db',
'state.sqlite',
'manifest.json',
join('replays', 'replay.memory-flow.v1.json'),
join('raw-sources', 'warehouse', 'accounts.csv'),
join('raw-sources', 'dbt', 'schema.yml'),
join('raw-sources', 'bi', 'revenue_exec.dashboard.lookml'),
join('raw-sources', 'notion', 'revenue-reporting-policy.md'),
join('semantic-layer', 'orbit_demo', 'accounts.yaml'),
join('knowledge', 'global', 'arr-contract-first.md'),
join('links', 'provenance.json'),
join('reports', 'seeded-demo-report.json'),
] as const;
async function exists(path: string): Promise<boolean> {
try {
await access(path, fsConstants.F_OK);
return true;
} catch {
return false;
}
}
async function loadSeededManifest(projectDir: string): Promise<DemoSeededManifest> {
const raw = await readFile(join(projectDir, 'manifest.json'), 'utf-8');
return JSON.parse(raw) as DemoSeededManifest;
}
async function listFilesInDir(dir: string, ext?: string): Promise<string[]> {
try {
const entries = await readdir(dir, { recursive: true });
return entries
.filter((entry): entry is string => typeof entry === 'string')
.filter((entry) => !ext || entry.endsWith(ext))
.sort();
} catch {
return [];
}
}
async function inspectSeededProjectStatus(projectDir: string): Promise<{ status: SeededInspectReadiness; missing: string[] }> {
const missing: string[] = [];
for (const relativePath of REQUIRED_SEEDED_PROJECT_PATHS) {
if (!(await exists(join(projectDir, relativePath)))) {
missing.push(relativePath);
}
}
if (missing.length === REQUIRED_SEEDED_PROJECT_PATHS.length) {
return { status: 'missing', missing };
}
if (missing.length > 0) {
return { status: 'corrupt', missing };
}
return { status: 'ready', missing: [] };
}
async function loadLinksCount(projectDir: string): Promise<number> {
try {
const raw = await readFile(join(projectDir, 'links', 'provenance.json'), 'utf-8');
const links = JSON.parse(raw) as unknown[];
return links.length;
} catch {
return 0;
}
}
async function loadSeededReplay(projectDir: string): Promise<MemoryFlowReplayInput | null> {
const latest = await loadLatestDemoReplay(projectDir);
if (latest) {
return latest;
}
try {
return await loadPackagedDemoReplay();
} catch {
return null;
}
}
function sourceBundleFromManifest(manifest: DemoSeededManifest): SeededInspectSummary['sourceBundle'] {
const warehouse = manifest.sources.warehouse;
const rowCounts = Object.fromEntries(Object.entries(warehouse.rowCounts).sort(([a], [b]) => a.localeCompare(b)));
const totalRows = Object.values(rowCounts).reduce((total, count) => total + count, 0);
return {
warehouse: {
label: warehouse.label,
path: warehouse.path ?? 'demo.db',
tableCount: warehouse.tables,
rowCounts,
totalRows,
},
dbt: {
label: manifest.sources.dbt.label,
path: manifest.sources.dbt.path ?? 'raw-sources/dbt',
modelCount: manifest.sources.dbt.models,
sourceTableCount: manifest.sources.dbt.sourceTables,
},
bi: {
label: manifest.sources.bi.label,
path: manifest.sources.bi.path ?? 'raw-sources/bi',
exploreCount: manifest.sources.bi.explores,
dashboardCount: manifest.sources.bi.dashboards,
},
notion: {
label: manifest.sources.notion.label,
path: manifest.sources.notion.path ?? 'raw-sources/notion',
pageCount: manifest.sources.notion.pages,
},
};
}
function nextCommands(): SeededInspectSummary['nextCommands'] {
return [...KLO_NEXT_STEP_COMMANDS];
}
function modeMetadataFromReplay(replay: MemoryFlowReplayInput | null): SeededInspectSummary['modeMetadata'] {
return {
mode: 'seeded',
source: 'packaged demo project',
generatedContext: 'prebuilt from bundled assets',
llmCalls: 'none',
origin: replay?.metadata?.origin ?? 'packaged',
timing: replay?.metadata?.timing ?? 'prebuilt',
sourceReportId: replay?.metadata?.sourceReportId ?? 'demo-seeded-report',
sourceReportPath: replay?.metadata?.sourceReportPath ?? 'reports/seeded-demo-report.json',
};
}
export async function inspectSeededProject(projectDir: string): Promise<SeededInspectSummary> {
const root = resolve(projectDir);
const manifest = await loadSeededManifest(root);
const latestReplay = await loadSeededReplay(root);
const semanticLayerPath = manifest.generated.semanticLayer.path ?? 'semantic-layer/orbit_demo';
const knowledgePath = manifest.generated.knowledge.path ?? 'knowledge/global';
const linksPath = join(manifest.generated.links.path ?? 'links', 'provenance.json');
const reportFiles = await listFilesInDir(join(root, 'reports'), '.json');
const replayFiles = await listFilesInDir(join(root, 'replays'), '.json');
return {
projectDir: root,
mode: 'seeded',
manifest,
status: await inspectSeededProjectStatus(root),
sourceBundle: sourceBundleFromManifest(manifest),
generatedOutputs: {
semanticLayer: {
path: semanticLayerPath,
manifestSourceCount: manifest.generated.semanticLayer.sourceCount,
fileCount: (await listFilesInDir(join(root, semanticLayerPath), '.yaml')).length,
},
knowledge: {
path: knowledgePath,
manifestPageCount: manifest.generated.knowledge.pageCount,
fileCount: (await listFilesInDir(join(root, knowledgePath), '.md')).length,
},
links: {
path: linksPath,
manifestLinkCount: manifest.generated.links.linkCount,
linkCount: await loadLinksCount(root),
},
reports: {
primaryPath: reportFiles[0] ? join('reports', reportFiles[0]) : 'reports/seeded-demo-report.json',
fileCount: reportFiles.length,
},
replays: {
primaryPath: join('replays', 'replay.memory-flow.v1.json'),
latestPath: join('replays', DEMO_LATEST_REPLAY_FILE),
fileCount: replayFiles.length,
},
},
modeMetadata: modeMetadataFromReplay(latestReplay),
nextCommands: nextCommands(),
latestReplay,
};
}
function rowCountPreview(rowCounts: Record<string, number>): string {
return Object.entries(rowCounts)
.map(([name, count]) => `${name} ${count}`)
.join(', ');
}
function replayLine(summary: SeededInspectSummary): string {
const metadata = summary.latestReplay?.metadata ?? summary.modeMetadata;
return `Latest replay: ${metadata.mode} (${metadata.origin}, ${metadata.timing})`;
}
export function formatSeededInspect(summary: SeededInspectSummary): string {
const source = summary.sourceBundle;
const generated = summary.generatedOutputs;
const lines = [`Demo project: ${summary.projectDir}`, `Status: ${summary.status.status}`];
if (summary.status.missing.length > 0) {
lines.push(`Missing: ${summary.status.missing.join(', ')}`);
}
lines.push(
`Mode: seeded (pre-seeded demo project)`,
`Source: ${summary.modeMetadata.source}`,
`Generated context: ${summary.modeMetadata.generatedContext}`,
`LLM calls: ${summary.modeMetadata.llmCalls}`,
'',
'Source bundle:',
` Warehouse: ${source.warehouse.tableCount} tables, ${source.warehouse.totalRows.toLocaleString()} rows`,
` Rows: ${rowCountPreview(source.warehouse.rowCounts)}`,
` dbt: ${source.dbt.modelCount} models, ${source.dbt.sourceTableCount} source tables`,
` BI: ${source.bi.exploreCount} explores, ${source.bi.dashboardCount} dashboards`,
` Notion: ${source.notion.pageCount} pages`,
'',
'Generated context:',
` Semantic-layer sources: ${generated.semanticLayer.manifestSourceCount} manifest, ${generated.semanticLayer.fileCount} files`,
` Knowledge pages: ${generated.knowledge.manifestPageCount} manifest, ${generated.knowledge.fileCount} files`,
` Evidence links: ${generated.links.manifestLinkCount} manifest, ${generated.links.linkCount} links`,
'',
`Report: ${generated.reports.primaryPath}`,
`Replay: ${generated.replays.primaryPath}`,
replayLine(summary),
'',
'What to do next:',
);
for (const command of summary.nextCommands) {
lines.push(` $ ${command.command.padEnd(KLO_NEXT_STEP_COMMAND_WIDTH)} ${command.description}`);
}
lines.push('', `Your KLO project files are at: ${summary.projectDir}`, '');
return lines.join('\n');
}

View file

@ -0,0 +1,117 @@
import { access, readFile, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { afterEach, describe, expect, it } from 'vitest';
import { ensureSeededDemoProject } from './demo-assets.js';
import { runDemoSeeded } from './demo-seeded.js';
describe('demo seeded mode', () => {
const projectDir = join(tmpdir(), `klo-demo-seeded-${process.pid}`);
afterEach(async () => {
await rm(projectDir, { recursive: true, force: true });
});
it('hydrates a complete seeded project with all asset directories', async () => {
const result = await ensureSeededDemoProject({ projectDir, force: false });
expect(result.projectDir).toBe(projectDir);
await expect(access(join(projectDir, 'demo.db'))).resolves.toBeUndefined();
await expect(access(join(projectDir, 'klo.yaml'))).resolves.toBeUndefined();
await expect(access(join(projectDir, 'manifest.json'))).resolves.toBeUndefined();
await expect(access(join(projectDir, 'semantic-layer/orbit_demo/accounts.yaml'))).resolves.toBeUndefined();
await expect(access(join(projectDir, 'knowledge/global/arr-contract-first.md'))).resolves.toBeUndefined();
await expect(access(join(projectDir, 'raw-sources/dbt/schema.yml'))).resolves.toBeUndefined();
await expect(access(join(projectDir, 'raw-sources/bi/revenue_exec.dashboard.lookml'))).resolves.toBeUndefined();
await expect(access(join(projectDir, 'raw-sources/notion/revenue-reporting-policy.md'))).resolves.toBeUndefined();
await expect(access(join(projectDir, 'links/provenance.json'))).resolves.toBeUndefined();
await expect(access(join(projectDir, 'reports/seeded-demo-report.json'))).resolves.toBeUndefined();
});
it('does not load or call any LLM provider in seeded mode', async () => {
const result = await runDemoSeeded({ projectDir });
expect(result.replay.metadata?.mode).toBe('seeded');
expect(result.replay.metadata?.timing).toBe('prebuilt');
expect(result.inspect.mode).toBe('seeded');
const config = await readFile(join(projectDir, 'klo.yaml'), 'utf-8');
expect(config).toContain('api_key: env:ANTHROPIC_API_KEY');
expect(config).not.toContain('sk-ant-');
});
it('creates the project under /tmp by default', async () => {
const result = await runDemoSeeded({ projectDir });
expect(result.projectDir).toBe(projectDir);
});
it('replay metadata identifies mode honestly', async () => {
const result = await runDemoSeeded({ projectDir });
expect(result.replay.metadata).toMatchObject({
mode: 'seeded',
origin: 'packaged',
timing: 'prebuilt',
});
expect(result.replay.runId).toBe('demo-seeded-orbit');
});
it('packaged seeded replay is honest and shows every source family', async () => {
const result = await runDemoSeeded({ projectDir });
const sourceEvents = result.replay.events.filter((event) => event.type === 'source_acquired');
const adapters = sourceEvents.map((event) => event.adapter).sort();
expect(result.replay.metadata).toMatchObject({
mode: 'seeded',
origin: 'packaged',
timing: 'prebuilt',
sourceReportPath: 'reports/seeded-demo-report.json',
});
expect(adapters).toEqual(['dbt_descriptions', 'live-database', 'looker', 'notion']);
expect(result.replay.events).not.toContainEqual(
expect.objectContaining({ type: 'stage_skipped', reason: expect.stringContaining('deterministic') }),
);
expect(JSON.stringify(result.replay)).not.toContain('LLM ran');
});
it('seeded animation shows all demo source families', async () => {
const result = await runDemoSeeded({ projectDir });
const adapters = result.replay.events
.filter((e) => e.type === 'source_acquired')
.map((e) => (e as { adapter: string }).adapter);
expect(adapters).toContain('live-database');
expect(adapters).toContain('dbt_descriptions');
expect(adapters).toContain('looker');
expect(adapters).toContain('notion');
});
it('SL YAML validates correctly', async () => {
await ensureSeededDemoProject({ projectDir, force: false });
const slYaml = await readFile(join(projectDir, 'semantic-layer/orbit_demo/accounts.yaml'), 'utf-8');
expect(slYaml).toContain('name: accounts');
expect(slYaml).toContain('grain:');
expect(slYaml).toContain('columns:');
expect(slYaml).toContain('measures:');
expect(slYaml).toContain('joins:');
});
it('wiki pages have valid frontmatter', async () => {
await ensureSeededDemoProject({ projectDir, force: false });
const wiki = await readFile(join(projectDir, 'knowledge/global/arr-contract-first.md'), 'utf-8');
expect(wiki).toContain('---');
expect(wiki).toContain('summary:');
expect(wiki).toContain('tags:');
expect(wiki).toContain('sl_refs:');
expect(wiki).toContain('usage_mode: auto');
});
it('links are searchable through provenance file', async () => {
await ensureSeededDemoProject({ projectDir, force: false });
const raw = await readFile(join(projectDir, 'links/provenance.json'), 'utf-8');
const links = JSON.parse(raw) as Array<{ id: string; artifactKind: string }>;
expect(links.length).toBe(23);
expect(links.some((l) => l.artifactKind === 'wiki')).toBe(true);
expect(links.some((l) => l.artifactKind === 'sl')).toBe(true);
});
});

View file

@ -0,0 +1,41 @@
import type { MemoryFlowReplayInput } from '@klo/context/ingest/memory-flow';
import {
ensureSeededDemoProject,
loadPackagedDemoReplay,
} from './demo-assets.js';
import { writeDemoReplay } from './demo-replay-store.js';
import { inspectSeededProject, type SeededInspectSummary } from './demo-seeded-inspect.js';
export {
formatSeededInspect,
inspectSeededProject,
type DemoSeededManifest,
type SeededInspectSummary,
} from './demo-seeded-inspect.js';
export interface DemoSeededResult {
projectDir: string;
replay: MemoryFlowReplayInput;
inspect: SeededInspectSummary;
}
export async function runDemoSeeded(options: {
projectDir: string;
}): Promise<DemoSeededResult> {
const result = await ensureSeededDemoProject({ projectDir: options.projectDir, force: false });
const replay = await loadPackagedDemoReplay();
const replayWithDir: MemoryFlowReplayInput = {
...replay,
sourceDir: result.projectDir,
};
await writeDemoReplay(result.projectDir, replayWithDir, { label: 'seeded' });
const inspect = await inspectSeededProject(result.projectDir);
return {
projectDir: result.projectDir,
replay: replayWithDir,
inspect,
};
}

View file

@ -0,0 +1,751 @@
import { mkdtemp, readFile, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import type { IngestReportSnapshot, MemoryFlowReplayInput } from '@klo/context/ingest';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { runKloDemo } from './demo.js';
import { DEMO_FULL_JOB_ID, defaultDemoProjectDir, ensureDemoProject } from './demo-assets.js';
import type { DemoFullResult } from './demo-full.js';
import { createTestDemoPromptAdapter } from './demo-interaction.js';
import type { renderMemoryFlowTui } from './memory-flow-tui.js';
import { KLO_NEXT_STEP_COMMANDS } from './next-steps.js';
import { resetVizFallbackWarningsForTest } from './viz-fallback.js';
function makeIo(options: { isTTY?: boolean; columns?: number; rawMode?: boolean } = {}) {
let stdout = '';
let stderr = '';
return {
io: {
stdin: {
isTTY: options.isTTY ?? false,
...(options.rawMode === false ? {} : { setRawMode: vi.fn() }),
},
stdout: {
isTTY: options.isTTY ?? false,
columns: options.columns ?? 140,
write: (chunk: string) => {
stdout += chunk;
},
},
stderr: {
write: (chunk: string) => {
stderr += chunk;
},
},
},
stdout: () => stdout,
stderr: () => stderr,
};
}
function fakeFullResult(projectDir: string): DemoFullResult {
const report: IngestReportSnapshot = {
id: 'report-full',
runId: 'run-full',
jobId: DEMO_FULL_JOB_ID,
connectionId: 'orbit_demo',
sourceKey: 'live-database',
createdAt: '2026-05-01T00:00:00.000Z',
body: {
syncId: 'sync-full',
diffSummary: { added: 7, modified: 0, deleted: 0, unchanged: 0 },
commitSha: null,
workUnits: [
{
unitKey: 'accounts',
rawFiles: ['accounts.schema.json'],
status: 'success',
actions: [
{ target: 'wiki', type: 'created', key: 'knowledge/accounts.md', detail: 'account lifecycle context' },
{ target: 'sl', type: 'created', key: 'orbit_demo.accounts', detail: 'accounts semantic source' },
],
touchedSlSources: [{ connectionId: 'orbit_demo', sourceName: 'orbit_demo.accounts' }],
},
],
failedWorkUnits: [],
reconciliationSkipped: false,
conflictsResolved: [],
evictionsApplied: [],
unmappedFallbacks: [],
evictionInputs: [],
unresolvedCards: [],
supersededBy: null,
overrideOf: null,
provenanceRows: [
{
rawPath: 'accounts.schema.json',
artifactKind: 'wiki',
artifactKey: 'knowledge/accounts.md',
actionType: 'wiki_written',
},
],
toolTranscripts: [],
},
};
return {
project: { projectDir } as never,
scan: { report: { runId: 'scan-run' } } as never,
ingest: { result: { ok: true }, report } as never,
report,
replay: {
runId: 'run-full',
connectionId: 'orbit_demo',
adapter: 'live-database',
status: 'done',
sourceDir: `${projectDir}/raw-sources/orbit_demo/live-database/sync-full`,
syncId: 'sync-full',
errors: [],
events: [
{ type: 'source_acquired', adapter: 'live-database', trigger: 'demo_full', fileCount: 7 },
{ type: 'saved', commitSha: null, wikiCount: 1, slCount: 1 },
{ type: 'provenance_recorded', rowCount: 1 },
{ type: 'report_created', runId: 'run-full', reportPath: 'report-full' },
],
plannedWorkUnits: [],
details: { actions: [], provenance: [], transcripts: [] },
},
};
}
describe('runKloDemo', () => {
let tempDir: string;
beforeEach(async () => {
resetVizFallbackWarningsForTest();
tempDir = await mkdtemp(join(tmpdir(), 'klo-demo-command-'));
});
afterEach(async () => {
await rm(tempDir, { recursive: true, force: true });
});
it('initializes the demo project', async () => {
const io = makeIo();
await expect(
runKloDemo({ command: 'init', projectDir: tempDir, force: false, inputMode: 'disabled' }, io.io),
).resolves.toBe(0);
expect(io.stdout()).toContain(`Demo project: ${tempDir}`);
expect(io.stdout()).toContain('Config:');
expect(io.stdout()).toContain('Replay:');
expect(io.stderr()).toBe('');
});
it('renders the packaged replay in no-input viz mode', async () => {
const io = makeIo({ isTTY: true });
await expect(
runKloDemo(
{ command: 'replay', projectDir: tempDir, outputMode: 'viz', inputMode: 'disabled' },
io.io,
{ env: { ...process.env, TERM: 'xterm-256color' } },
),
).resolves.toBe(0);
expect(io.stdout()).toContain('KLO memory flow Warehouse + dbt + BI + Docs done');
expect(io.stdout()).toContain('Saved 16 memories');
expect(io.stderr()).toBe('');
});
it('routes interactive packaged replay viz through the stored TUI renderer', async () => {
const io = makeIo({ isTTY: true });
const renderStoredMemoryFlow = vi.fn<typeof renderMemoryFlowTui>(async () => true);
await expect(
runKloDemo(
{ command: 'replay', projectDir: tempDir, outputMode: 'viz' },
io.io,
{ env: { ...process.env, TERM: 'xterm-256color' }, renderStoredMemoryFlow },
),
).resolves.toBe(0);
expect(renderStoredMemoryFlow).toHaveBeenCalledTimes(1);
expect(renderStoredMemoryFlow.mock.calls[0]?.[0]).toMatchObject({
runId: 'demo-seeded-orbit',
connectionId: 'orbit_demo',
adapter: 'live-database',
});
expect(renderStoredMemoryFlow.mock.calls[0]?.[2]).toEqual({ speedMultiplier: 0.125 });
expect(io.stdout()).toContain('KLO finished ingesting your data');
expect(io.stderr()).toBe('');
});
it('routes interactive seeded demo viz through the stored TUI renderer at eighth speed', async () => {
const io = makeIo({ isTTY: true });
const renderStoredMemoryFlow = vi.fn<typeof renderMemoryFlowTui>(async () => true);
await expect(
runKloDemo(
{ command: 'seeded', projectDir: tempDir, outputMode: 'viz' },
io.io,
{ env: { ...process.env, TERM: 'xterm-256color' }, renderStoredMemoryFlow },
),
).resolves.toBe(0);
expect(renderStoredMemoryFlow).toHaveBeenCalledTimes(1);
expect(renderStoredMemoryFlow.mock.calls[0]?.[2]).toEqual({ speedMultiplier: 0.125 });
expect(io.stdout()).toContain('KLO finished ingesting your data');
expect(io.stderr()).toBe('');
});
it('falls back to plain replay output when interactive replay viz lacks stdin raw mode', async () => {
const io = makeIo({ isTTY: true, rawMode: false });
const renderStoredMemoryFlow = vi.fn(async (_input: MemoryFlowReplayInput, _io: unknown) => true);
await expect(
runKloDemo(
{ command: 'replay', projectDir: tempDir, outputMode: 'viz' },
io.io,
{ env: { ...process.env, TERM: 'xterm-256color' }, renderStoredMemoryFlow },
),
).resolves.toBe(0);
expect(renderStoredMemoryFlow).not.toHaveBeenCalled();
expect(io.stdout()).toContain('Memory-flow summary: done');
expect(io.stdout()).toContain('Connection: orbit_demo');
expect(io.stdout()).toContain('klo sl list');
expect(io.stdout()).toContain('klo wiki list');
expect(io.stdout()).toContain('klo serve --mcp stdio --user-id local');
expect(io.stdout()).not.toContain('KLO memory flow');
expect(io.stderr()).toContain(
'Visualization requested but stdin raw mode is unavailable; printing plain output.',
);
});
it('degrades default visual demo replay to a plain memory-flow summary when stdout is redirected', async () => {
const testIo = makeIo({ isTTY: false });
await expect(
runKloDemo({ command: 'replay', projectDir: tempDir, outputMode: 'viz', inputMode: 'disabled' }, testIo.io),
).resolves.toBe(0);
expect(testIo.stdout()).toContain('Memory-flow summary: done');
expect(testIo.stdout()).toContain('Connection: orbit_demo');
expect(testIo.stdout()).toContain('klo sl list');
expect(testIo.stdout()).toContain('klo wiki list');
expect(testIo.stdout()).toContain('klo serve --mcp stdio --user-id local');
expect(testIo.stdout()).not.toContain('KLO memory flow');
expect(testIo.stderr()).toContain(
'Visualization requested but stdout is not an interactive terminal; printing plain output.',
);
});
it('prints JSON replay output when requested', async () => {
const io = makeIo();
await expect(
runKloDemo({ command: 'replay', projectDir: tempDir, outputMode: 'json', inputMode: 'disabled' }, io.io),
).resolves.toBe(0);
expect(JSON.parse(io.stdout())).toMatchObject({ runId: 'demo-seeded-orbit', connectionId: 'orbit_demo' });
expect(io.stderr()).toBe('');
});
it('runs the packaged SQLite demo scan', async () => {
const io = makeIo();
await expect(runKloDemo({ command: 'scan', projectDir: tempDir, inputMode: 'disabled' }, io.io)).resolves.toBe(0);
expect(io.stdout()).toContain('Demo scan: done');
expect(io.stdout()).toContain('Connection: orbit_demo');
expect(io.stdout()).toContain('Driver: sqlite');
expect(io.stdout()).toContain('Report: raw-sources/orbit_demo/live-database/');
expect(io.stderr()).toBe('');
});
it('runs seeded mode with pre-seeded assets and inspect summary', async () => {
const io = makeIo({ isTTY: true });
await expect(
runKloDemo(
{ command: 'seeded', projectDir: tempDir, outputMode: 'plain', inputMode: 'disabled' },
io.io,
{ env: { ...process.env, TERM: 'xterm-256color' } },
),
).resolves.toBe(0);
expect(io.stdout()).toContain('Mode: seeded');
expect(io.stdout()).toContain('LLM calls: none');
expect(io.stdout()).toContain('Semantic-layer sources:');
expect(io.stdout()).toContain('Knowledge pages:');
expect(io.stderr()).toBe('');
});
it('uses seeded mode as the default demo and creates a temp project when no project-dir is supplied', async () => {
const io = makeIo();
await expect(
runKloDemo(
{ command: 'seeded', projectDir: defaultDemoProjectDir(), outputMode: 'plain', inputMode: 'disabled' },
io.io,
),
).resolves.toBe(0);
expect(io.stdout()).toContain('Mode: seeded');
expect(io.stdout()).toContain('Source: packaged demo project');
expect(io.stdout()).toContain('Generated context: prebuilt from bundled assets');
expect(io.stdout()).toContain('LLM calls: none');
expect(io.stdout()).toContain('Your KLO project files are at:');
expect(io.stdout()).toContain(join(tmpdir(), 'klo-demo-'));
expect(io.stdout()).toContain('klo serve --mcp stdio');
expect(io.stdout()).not.toContain(['klo', 'mcp'].join(' '));
expect(io.stdout()).not.toContain('deterministic');
});
it('degrades default visual seeded demo to plain output when TERM is dumb', async () => {
const testIo = makeIo({ isTTY: true, columns: 120 });
await expect(
runKloDemo(
{ command: 'seeded', projectDir: tempDir, outputMode: 'viz', inputMode: 'disabled' },
testIo.io,
{ env: { ...process.env, TERM: 'dumb' } },
),
).resolves.toBe(0);
expect(testIo.stdout()).toContain('Mode: seeded');
expect(testIo.stdout()).toContain('LLM calls: none');
expect(testIo.stderr()).toContain(
'Visualization requested but TERM=dumb does not support the visual renderer; printing plain output.',
);
});
it('prints demo inspect as plain text and JSON', async () => {
const seededIo = makeIo();
await expect(
runKloDemo({ command: 'seeded', projectDir: tempDir, outputMode: 'plain', inputMode: 'disabled' }, seededIo.io),
).resolves.toBe(0);
const plainIo = makeIo();
await expect(
runKloDemo({ command: 'inspect', projectDir: tempDir, outputMode: 'plain', inputMode: 'disabled' }, plainIo.io),
).resolves.toBe(0);
expect(plainIo.stdout()).toContain('Mode: seeded');
expect(plainIo.stdout()).toContain('Semantic-layer sources:');
const jsonIo = makeIo();
await expect(
runKloDemo({ command: 'inspect', projectDir: tempDir, outputMode: 'json', inputMode: 'disabled' }, jsonIo.io),
).resolves.toBe(0);
const parsed = JSON.parse(jsonIo.stdout());
expect(parsed).toMatchObject({
projectDir: tempDir,
mode: 'seeded',
status: { status: 'ready', missing: [] },
sourceBundle: {
warehouse: { tableCount: 8, totalRows: 11234 },
dbt: { modelCount: 3, sourceTableCount: 8 },
bi: { exploreCount: 5, dashboardCount: 2 },
notion: { pageCount: 8 },
},
generatedOutputs: {
semanticLayer: { manifestSourceCount: 6, fileCount: 6 },
knowledge: { manifestPageCount: 10, fileCount: 10 },
links: { manifestLinkCount: 23, linkCount: 23 },
reports: { primaryPath: 'reports/seeded-demo-report.json', fileCount: 1 },
},
modeMetadata: {
mode: 'seeded',
source: 'packaged demo project',
generatedContext: 'prebuilt from bundled assets',
llmCalls: 'none',
},
nextCommands: KLO_NEXT_STEP_COMMANDS,
});
expect(parsed.generatedOutputs.replays.fileCount).toBeGreaterThanOrEqual(3);
expect(jsonIo.stderr()).toBe('');
});
it('routes top-level full mode and prints memory-flow plus final summary', async () => {
const testIo = makeIo({ isTTY: true });
const runFullDemo = vi.fn().mockResolvedValue(fakeFullResult(tempDir));
await ensureDemoProject({ projectDir: tempDir, force: false });
await expect(
runKloDemo({ command: 'full', projectDir: tempDir, outputMode: 'viz', inputMode: 'disabled' }, testIo.io, {
env: {},
runFullDemo,
}),
).resolves.toBe(0);
expect(runFullDemo).toHaveBeenCalledWith(
expect.objectContaining({
projectDir: tempDir,
env: {},
onMemoryFlowChange: expect.any(Function),
}),
);
expect(testIo.stdout()).toContain('KLO memory flow orbit_demo/live-database done');
expect(testIo.stdout()).toContain('Full demo ingest: done');
expect(testIo.stdout()).toContain('Next: klo setup demo inspect');
expect(testIo.stdout()).toContain('Shows the files, semantic-layer sources, and memory KLO just produced.');
});
it('streams live memory-flow snapshots for full demo viz and then prints final summary', async () => {
const testIo = makeIo({ isTTY: true, columns: 120 });
const liveSession = {
update: vi.fn(),
close: vi.fn(),
isClosed: vi.fn(() => false),
};
const startLiveMemoryFlow = vi.fn(async (_input: MemoryFlowReplayInput, _io: unknown) => liveSession);
const runFullDemo = vi.fn(
async (options: { projectDir: string; onMemoryFlowChange?: (snapshot: MemoryFlowReplayInput) => void }) => {
options.onMemoryFlowChange?.({
...fakeFullResult(tempDir).replay,
status: 'running',
events: [{ type: 'source_acquired', adapter: 'live-database', trigger: 'demo_full', fileCount: 7 }],
});
return fakeFullResult(tempDir);
},
);
await ensureDemoProject({ projectDir: tempDir, force: false });
await expect(
runKloDemo({ command: 'full', projectDir: tempDir, outputMode: 'viz' }, testIo.io, {
env: { ANTHROPIC_API_KEY: 'sk-ant-test' }, // pragma: allowlist secret
prompts: createTestDemoPromptAdapter({ choices: ['reuse'] }),
runFullDemo,
startLiveMemoryFlow,
}),
).resolves.toBe(0);
expect(startLiveMemoryFlow).toHaveBeenCalledTimes(1);
expect(liveSession.update).toHaveBeenCalledTimes(1);
expect(liveSession.close).toHaveBeenCalledTimes(1);
expect(testIo.stdout()).not.toContain('Memory-flow summary: done');
expect(testIo.stdout()).toContain('KLO finished ingesting your data');
expect(testIo.stdout()).toContain('klo sl list');
expect(testIo.stdout()).toContain('klo wiki list');
expect(testIo.stdout()).toContain('klo serve --mcp stdio --user-id local');
expect(testIo.stdout()).not.toContain(['klo', 'ask'].join(' '));
expect(testIo.stdout()).not.toContain(['klo', 'mcp'].join(' '));
});
it('uses plain progress for full demo viz when stdin raw mode is unavailable', async () => {
const testIo = makeIo({ isTTY: true, rawMode: false, columns: 120 });
const liveSession = {
update: vi.fn(),
close: vi.fn(),
isClosed: vi.fn(() => false),
};
const startLiveMemoryFlow = vi.fn(async (_input: MemoryFlowReplayInput, _io: unknown) => liveSession);
const runFullDemo = vi.fn(
async (options: { projectDir: string; onMemoryFlowChange?: (snapshot: MemoryFlowReplayInput) => void }) => {
options.onMemoryFlowChange?.({
...fakeFullResult(tempDir).replay,
status: 'running',
events: [{ type: 'source_acquired', adapter: 'live-database', trigger: 'demo_full', fileCount: 7 }],
});
return fakeFullResult(tempDir);
},
);
await ensureDemoProject({ projectDir: tempDir, force: false });
await expect(
runKloDemo({ command: 'full', projectDir: tempDir, outputMode: 'viz' }, testIo.io, {
env: { ANTHROPIC_API_KEY: 'sk-ant-test' }, // pragma: allowlist secret
prompts: createTestDemoPromptAdapter({ choices: ['reuse'] }),
runFullDemo,
startLiveMemoryFlow,
}),
).resolves.toBe(0);
expect(startLiveMemoryFlow).not.toHaveBeenCalled();
expect(runFullDemo).toHaveBeenCalledWith(
expect.objectContaining({
onMemoryFlowChange: expect.any(Function),
}),
);
expect(testIo.stdout()).toContain('[connect] Connected live-database - 7 database files (demo_full)');
expect(testIo.stdout()).toContain('Full demo ingest: done');
expect(testIo.stdout()).not.toContain('KLO memory flow');
expect(testIo.stderr()).toContain(
'Visualization requested but stdin raw mode is unavailable; printing plain output.',
);
});
it('streams plain-text progress lines for full demo when no live TUI is active', async () => {
const testIo = makeIo();
const runFullDemo = vi.fn(
async (options: { projectDir: string; onMemoryFlowChange?: (snapshot: MemoryFlowReplayInput) => void }) => {
const baseSnapshot = fakeFullResult(tempDir).replay;
options.onMemoryFlowChange?.({
...baseSnapshot,
status: 'running',
events: [{ type: 'source_acquired', adapter: 'live-database', trigger: 'manual_resync', fileCount: 7 }],
});
options.onMemoryFlowChange?.({
...baseSnapshot,
status: 'running',
events: [
{ type: 'source_acquired', adapter: 'live-database', trigger: 'manual_resync', fileCount: 7 },
{ type: 'diff_computed', added: 0, modified: 0, deleted: 0, unchanged: 7 },
],
});
return fakeFullResult(tempDir);
},
);
await ensureDemoProject({ projectDir: tempDir, force: false });
await expect(
runKloDemo(
{ command: 'full', projectDir: tempDir, outputMode: 'plain', inputMode: 'disabled' },
testIo.io,
{ env: { ANTHROPIC_API_KEY: 'sk-ant-test' }, runFullDemo }, // pragma: allowlist secret
),
).resolves.toBe(0);
const stdout = testIo.stdout();
expect(stdout).toContain('[connect] Connected live-database - 7 database files (manual_resync)');
expect(stdout).toContain('[diff] Tables: =7 unchanged');
expect(stdout).toContain('Full demo ingest: done');
});
it('skips plain progress lines for json output mode', async () => {
const testIo = makeIo();
const runFullDemo = vi.fn(
async (options: { projectDir: string; onMemoryFlowChange?: (snapshot: MemoryFlowReplayInput) => void }) => {
expect(options.onMemoryFlowChange).toBeUndefined();
return fakeFullResult(tempDir);
},
);
await ensureDemoProject({ projectDir: tempDir, force: false });
await expect(
runKloDemo(
{ command: 'full', projectDir: tempDir, outputMode: 'json', inputMode: 'disabled' },
testIo.io,
{ env: { ANTHROPIC_API_KEY: 'sk-ant-test' }, runFullDemo }, // pragma: allowlist secret
),
).resolves.toBe(0);
expect(testIo.stdout()).not.toContain('[connect]');
expect(testIo.stdout()).not.toContain('[snapshot]');
});
it('routes demo ingest full mode', async () => {
const testIo = makeIo();
const runFullDemo = vi.fn().mockResolvedValue(fakeFullResult(tempDir));
await ensureDemoProject({ projectDir: tempDir, force: false });
await expect(
runKloDemo(
{ command: 'ingest', mode: 'full', projectDir: tempDir, outputMode: 'plain', inputMode: 'disabled' },
testIo.io,
{ env: {}, runFullDemo },
),
).resolves.toBe(0);
expect(testIo.stdout()).toContain('Full demo ingest: done');
});
it('saves full-demo replay output for the next demo replay command', async () => {
const tempDir = await mkdtemp(join(tmpdir(), 'klo-demo-full-replay-'));
await ensureDemoProject({ projectDir: tempDir, force: false });
const io = makeIo();
await expect(
runKloDemo(
{ command: 'full', projectDir: tempDir, outputMode: 'plain', inputMode: 'disabled' },
io.io,
{
env: { ANTHROPIC_API_KEY: 'sk-ant-test' }, // pragma: allowlist secret
runFullDemo: vi.fn(async () => fakeFullResult(tempDir)),
},
),
).resolves.toBe(0);
const replayIo = makeIo();
await expect(
runKloDemo({ command: 'replay', projectDir: tempDir, outputMode: 'json', inputMode: 'disabled' }, replayIo.io),
).resolves.toBe(0);
expect(JSON.parse(replayIo.stdout())).toMatchObject({
runId: 'run-full',
metadata: { mode: 'full', origin: 'captured' },
});
});
it('routes demo ingest seeded mode through the seeded path', async () => {
const testIo = makeIo();
await expect(
runKloDemo(
{ command: 'ingest', mode: 'seeded', projectDir: tempDir, outputMode: 'plain', inputMode: 'disabled' },
testIo.io,
),
).resolves.toBe(0);
expect(testIo.stdout()).toContain('Mode: seeded');
expect(testIo.stdout()).toContain('LLM calls: none');
});
it('routes demo doctor through the doctor module', async () => {
const testIo = makeIo();
const runDoctor = vi.fn().mockResolvedValue(0);
await expect(
runKloDemo(
{
command: 'doctor',
projectDir: tempDir,
outputMode: 'plain',
inputMode: 'disabled',
},
testIo.io,
{ runDoctor },
),
).resolves.toBe(0);
expect(runDoctor).toHaveBeenCalledWith(
{
command: 'demo',
projectDir: tempDir,
outputMode: 'plain',
inputMode: 'disabled',
},
testIo.io,
);
});
it('resets the demo project only when force is explicit', async () => {
await ensureDemoProject({ projectDir: tempDir, force: false });
await rm(join(tempDir, 'demo.db'), { force: true });
const rejected = makeIo();
await expect(
runKloDemo({ command: 'reset', projectDir: tempDir, force: false, inputMode: 'disabled' }, rejected.io),
).resolves.toBe(1);
expect(rejected.stderr()).toContain(`klo setup demo reset is destructive; pass --force to recreate ${tempDir}`);
const accepted = makeIo();
await expect(
runKloDemo({ command: 'reset', projectDir: tempDir, force: true, inputMode: 'disabled' }, accepted.io),
).resolves.toBe(0);
expect(accepted.stdout()).toContain(`Demo project reset: ${tempDir}`);
});
it('rehydrates seeded assets after reset --force', async () => {
const resetIo = makeIo();
await expect(
runKloDemo({ command: 'reset', projectDir: tempDir, force: true, inputMode: 'disabled' }, resetIo.io),
).resolves.toBe(0);
const seededIo = makeIo();
await expect(
runKloDemo(
{ command: 'seeded', projectDir: tempDir, outputMode: 'plain', inputMode: 'disabled' },
seededIo.io,
),
).resolves.toBe(0);
expect(seededIo.stdout()).toContain('Status: ready');
expect(seededIo.stdout()).toContain('Semantic-layer sources: 6 manifest, 6 files');
expect(seededIo.stdout()).toContain('Knowledge pages: 10 manifest, 10 files');
expect(seededIo.stdout()).not.toContain('Status: corrupt');
expect(seededIo.stdout()).not.toContain('Semantic-layer sources: 6 manifest, 0 files');
});
it('fails corrupted demo projects in no-input mode with reset guidance', async () => {
await ensureDemoProject({ projectDir: tempDir, force: false });
await rm(join(tempDir, 'demo.db'), { force: true });
const testIo = makeIo();
await expect(
runKloDemo({ command: 'replay', projectDir: tempDir, outputMode: 'plain', inputMode: 'disabled' }, testIo.io),
).resolves.toBe(1);
expect(testIo.stderr()).toContain(`Demo project is not ready at ${tempDir}: missing demo.db`);
expect(testIo.stderr()).toContain(`klo setup demo reset --project-dir ${tempDir} --force --no-input`);
});
it('uses a process-local Anthropic key from the interactive prompt', async () => {
const testIo = makeIo({ isTTY: true, columns: 120 });
const runFullDemo = vi.fn().mockResolvedValue(fakeFullResult(tempDir));
await ensureDemoProject({ projectDir: tempDir, force: false });
await expect(
runKloDemo(
{ command: 'full', projectDir: tempDir, outputMode: 'plain' },
testIo.io,
{
env: {},
prompts: createTestDemoPromptAdapter({
choices: ['reuse', 'process_key'],
passwords: ['sk-ant-process'], // pragma: allowlist secret
}),
runFullDemo,
},
),
).resolves.toBe(0);
expect(runFullDemo).toHaveBeenCalledWith(
expect.objectContaining({
projectDir: tempDir,
env: { ANTHROPIC_API_KEY: 'sk-ant-process' }, // pragma: allowlist secret
onMemoryFlowChange: expect.any(Function),
}),
);
expect(await readFile(join(tempDir, 'klo.yaml'), 'utf-8')).toContain('api_key: env:ANTHROPIC_API_KEY');
});
it('routes an interactive missing-key choice to seeded mode', async () => {
const testIo = makeIo({ isTTY: true, columns: 120 });
const runFullDemo = vi.fn();
await ensureDemoProject({ projectDir: tempDir, force: false });
await expect(
runKloDemo(
{ command: 'full', projectDir: tempDir, outputMode: 'plain' },
testIo.io,
{
env: {},
prompts: createTestDemoPromptAdapter({ choices: ['reuse', 'seeded'] }),
runFullDemo,
},
),
).resolves.toBe(0);
expect(runFullDemo).not.toHaveBeenCalled();
expect(testIo.stdout()).toContain('Mode: seeded');
expect(testIo.stdout()).toContain('LLM calls: none');
expect(testIo.stdout()).not.toContain('deterministic');
});
it('routes missing full-mode credentials to seeded when the interactive user chooses the no-LLM demo', async () => {
const testIo = makeIo({ isTTY: true });
await expect(
runKloDemo(
{ command: 'full', projectDir: tempDir, outputMode: 'plain' },
testIo.io,
{
env: { ...process.env, ANTHROPIC_API_KEY: '' },
prompts: createTestDemoPromptAdapter({ choices: ['seeded'] }),
},
),
).resolves.toBe(0);
expect(testIo.stdout()).toContain('Mode: seeded');
expect(testIo.stdout()).toContain('LLM calls: none');
expect(testIo.stdout()).not.toContain('deterministic');
});
it('routes an interactive missing-key choice to replay mode', async () => {
const testIo = makeIo({ isTTY: true, columns: 120 });
const runFullDemo = vi.fn();
await ensureDemoProject({ projectDir: tempDir, force: false });
await expect(
runKloDemo(
{ command: 'full', projectDir: tempDir, outputMode: 'viz' },
testIo.io,
{
env: {},
prompts: createTestDemoPromptAdapter({ choices: ['reuse', 'replay'] }),
runFullDemo,
},
),
).resolves.toBe(0);
expect(runFullDemo).not.toHaveBeenCalled();
expect(testIo.stdout()).toContain('KLO memory flow');
expect(testIo.stdout()).toContain('done');
});
});

544
packages/cli/src/demo.ts Normal file
View file

@ -0,0 +1,544 @@
import {
buildMemoryFlowViewModel,
formatMemoryFlowFinalSummary,
renderMemoryFlowReplay,
type MemoryFlowReplayInput,
} from '@klo/context/ingest/memory-flow';
import { resolveKloConfigReference } from '@klo/context/core';
import { loadKloProject } from '@klo/context/project';
import {
DEMO_ADAPTER,
DEMO_CONNECTION_ID,
DEMO_FULL_JOB_ID,
ensureDemoProject,
loadProjectDemoReplay,
resetDemoProject,
} from './demo-assets.js';
import { writeDemoReplay } from './demo-replay-store.js';
import {
formatDemoInspect,
formatDemoScanSummary,
inspectDemoProject,
runDemoScan,
} from './demo-scan.js';
import {
formatSeededInspect,
inspectSeededProject,
runDemoSeeded,
} from './demo-seeded.js';
import { buildFullDemoReplay, formatCleanDemoSummary, formatFullDemoSummary, fullDemoCredentialStatus, runDemoFull } from './demo-full.js';
import { createPlainProgressEmitter } from './demo-progress.js';
import {
chooseDemoProjectForInteractiveRun,
createClackDemoPromptAdapter,
resolveFullCredentialDecision,
type DemoPromptAdapter,
} from './demo-interaction.js';
import type { KloDoctorArgs } from './doctor.js';
import {
renderMemoryFlowTui,
startLiveMemoryFlowTui,
type KloMemoryFlowTuiIo,
type MemoryFlowTuiLiveSession,
} from './memory-flow-tui.js';
import {
rendererUnavailableVizFallback,
resolveVizFallback,
warnVizFallbackOnce,
} from './viz-fallback.js';
import { profileMark } from './startup-profile.js';
import { formatNextStepLines } from './next-steps.js';
profileMark('module:demo');
export type KloDemoOutputMode = 'plain' | 'json' | 'viz';
export type KloDemoInputMode = 'auto' | 'disabled';
export type KloDemoMode = 'full' | 'seeded';
export type KloDemoArgs =
| { command: 'init'; projectDir: string; force: boolean; inputMode?: KloDemoInputMode }
| { command: 'reset'; projectDir: string; force: boolean; inputMode?: KloDemoInputMode }
| { command: 'replay'; projectDir: string; outputMode: KloDemoOutputMode; inputMode?: KloDemoInputMode }
| { command: 'scan'; projectDir: string; inputMode?: KloDemoInputMode }
| { command: 'inspect'; projectDir: string; outputMode: KloDemoOutputMode; inputMode?: KloDemoInputMode }
| { command: 'doctor'; projectDir: string; outputMode: Exclude<KloDemoOutputMode, 'viz'>; inputMode?: KloDemoInputMode }
| { command: 'seeded'; projectDir: string; outputMode: KloDemoOutputMode; inputMode?: KloDemoInputMode }
| { command: 'full'; projectDir: string; outputMode: KloDemoOutputMode; inputMode?: KloDemoInputMode }
| {
command: 'ingest';
mode: KloDemoMode;
projectDir: string;
outputMode: KloDemoOutputMode;
inputMode?: KloDemoInputMode;
};
export interface KloDemoIo {
stdin?: KloMemoryFlowTuiIo['stdin'];
stdout: { isTTY?: boolean; columns?: number; write(chunk: string): void };
stderr: { write(chunk: string): void };
}
interface KloDemoDeps {
runFullDemo?: typeof runDemoFull;
runDoctor?: (args: KloDoctorArgs, io: KloDemoIo) => Promise<number>;
renderStoredMemoryFlow?: typeof renderMemoryFlowTui;
startLiveMemoryFlow?: typeof startLiveMemoryFlowTui;
env?: NodeJS.ProcessEnv;
prompts?: DemoPromptAdapter;
}
const ADAPTER_PREFIXES = ['live_database_', 'metabase_', 'looker_', 'lookml_', 'metricflow_', 'notion_', 'historic_sql_', 'dbt_descriptions_'];
const DEMO_TUI_SPEED_MULTIPLIER = 0.125;
function humanizeUnitKeyPlain(unitKey: string): string {
let key = unitKey.replace(/-/g, '_');
for (const prefix of ADAPTER_PREFIXES) {
if (key.startsWith(prefix)) { key = key.slice(prefix.length); break; }
}
return key.replace(/_/g, ' ');
}
function formatReplaySummary(input: MemoryFlowReplayInput): string {
let slCount = 0;
let wikiCount = 0;
let chunkCount = 0;
const unitResults: Array<{ unitKey: string; artifacts: Array<{ icon: string; text: string; hasSummary: boolean }> }> = [];
let currentUnit: { unitKey: string; artifacts: Array<{ icon: string; text: string; hasSummary: boolean }> } | null = null;
let conflictCount = 0;
for (const e of input.events) {
if (e.type === 'chunks_planned') {
chunkCount = e.chunkCount;
} else if (e.type === 'work_unit_started') {
currentUnit = { unitKey: e.unitKey, artifacts: [] };
} else if (e.type === 'candidate_action') {
if (e.target === 'sl') slCount++;
else wikiCount++;
const detail = input.details.actions.find((a) => a.key === e.key && a.unitKey === e.unitKey);
const icon = e.target === 'sl' ? '📊' : '📝';
const name = e.key.split('.').pop()?.replace(/[_-]/g, ' ') ?? e.key;
const text = detail?.summary ?? name;
currentUnit?.artifacts.push({ icon, text, hasSummary: !!detail?.summary });
} else if (e.type === 'work_unit_finished' && currentUnit) {
unitResults.push(currentUnit);
currentUnit = null;
} else if (e.type === 'reconciliation_finished') {
conflictCount = e.conflictCount;
}
}
const lines: string[] = ['', '★ KLO finished ingesting your data', ''];
if (chunkCount > 0) {
lines.push(` ✓ Analyzed ${chunkCount} business area${chunkCount === 1 ? '' : 's'}`);
}
lines.push(` ✓ Reconciled — ${conflictCount > 0 ? `${conflictCount} conflict${conflictCount === 1 ? '' : 's'} resolved` : 'no conflicts'}`);
lines.push('');
if (slCount > 0 || wikiCount > 0) {
lines.push(' KLO created:');
if (slCount > 0) lines.push(` 📊 ${slCount} query definition${slCount === 1 ? '' : 's'} — so agents can write accurate SQL for your data`);
if (wikiCount > 0) lines.push(` 📝 ${wikiCount} knowledge page${wikiCount === 1 ? '' : 's'} — so agents understand your business context`);
lines.push('');
}
const described = unitResults.flatMap((u) => u.artifacts).filter((a) => a.hasSummary);
for (const a of described) {
lines.push(` ${a.icon} ${a.text}`);
}
lines.push('');
lines.push(' What to do next:');
lines.push(...formatNextStepLines());
if (input.sourceDir) {
lines.push('');
lines.push(` Your KLO project files are at: ${input.sourceDir}`);
}
lines.push('');
return lines.join('\n');
}
function formatPlainReplaySummary(input: MemoryFlowReplayInput): string {
return [formatMemoryFlowFinalSummary(input).trimEnd(), '', 'What to do next:', ...formatNextStepLines(), ''].join('\n');
}
function writeReplay(input: MemoryFlowReplayInput, outputMode: KloDemoOutputMode, io: KloDemoIo): void {
if (outputMode === 'json') {
io.stdout.write(`${JSON.stringify(input, null, 2)}\n`);
return;
}
if (outputMode === 'plain') {
io.stdout.write(formatPlainReplaySummary(input));
return;
}
const view = buildMemoryFlowViewModel(input);
io.stdout.write(renderMemoryFlowReplay(view, { terminalWidth: io.stdout.columns ?? process.stdout.columns }));
}
async function writeStoredReplay(
input: MemoryFlowReplayInput,
outputMode: KloDemoOutputMode,
inputMode: KloDemoArgs['inputMode'],
io: KloDemoIo,
deps: KloDemoDeps,
env: NodeJS.ProcessEnv,
): Promise<void> {
const resolvedOutputMode = effectiveDemoOutputMode(outputMode, io, env, {
requireInput: inputMode !== 'disabled',
});
if (resolvedOutputMode !== 'viz') {
writeReplay(input, resolvedOutputMode, io);
return;
}
if (inputMode !== 'disabled') {
const renderStoredMemoryFlow = deps.renderStoredMemoryFlow ?? renderMemoryFlowTui;
if (
isTuiCapableDemoIo(io) &&
(await renderStoredMemoryFlow(input, io, { speedMultiplier: DEMO_TUI_SPEED_MULTIPLIER }))
) {
io.stdout.write(formatReplaySummary(input));
return;
}
}
writeReplay(input, resolvedOutputMode, io);
}
function writeInspect(
summary: Awaited<ReturnType<typeof inspectDemoProject>>,
outputMode: KloDemoOutputMode,
io: KloDemoIo,
): void {
if (outputMode === 'json') {
io.stdout.write(`${JSON.stringify(summary, null, 2)}\n`);
return;
}
io.stdout.write(formatDemoInspect(summary));
}
function writeFullDemo(
result: Awaited<ReturnType<typeof runDemoFull>>,
outputMode: KloDemoOutputMode,
io: KloDemoIo,
options: { liveWasRendered?: boolean; projectDir?: string } = {},
): void {
if (outputMode === 'json') {
io.stdout.write(`${JSON.stringify({ report: result.report, replay: result.replay }, null, 2)}\n`);
return;
}
if (outputMode === 'viz' && options.liveWasRendered !== true) {
writeReplay(buildFullDemoReplay(result.report), outputMode, io);
io.stdout.write('\n');
}
if (outputMode === 'viz' && options.liveWasRendered) {
io.stdout.write(formatCleanDemoSummary(result.report, options.projectDir ?? ''));
return;
}
if (outputMode === 'viz') {
io.stdout.write(formatMemoryFlowFinalSummary(buildFullDemoReplay(result.report)));
}
io.stdout.write(formatFullDemoSummary(result.report));
}
function replayWithFullMetadata(result: Awaited<ReturnType<typeof runDemoFull>>): MemoryFlowReplayInput {
if (result.replay.metadata) {
return result.replay;
}
return {
...result.replay,
metadata: {
schemaVersion: 1,
mode: 'full',
origin: 'captured',
timing: 'captured',
capturedAt: result.report.createdAt,
sourceReportId: result.report.id,
sourceReportPath: result.report.id,
fallbackReason: null,
},
reportId: result.replay.reportId ?? result.report.id,
reportPath: result.replay.reportPath ?? result.report.id,
};
}
function pickMemoryFlowProgress(
liveSession: MemoryFlowTuiLiveSession | null,
outputMode: KloDemoOutputMode,
io: KloDemoIo,
): ((snapshot: MemoryFlowReplayInput) => void) | undefined {
if (liveSession) {
return (snapshot: MemoryFlowReplayInput) => {
if (!liveSession.isClosed()) {
liveSession.update(snapshot);
}
};
}
if (outputMode === 'json') {
return undefined;
}
return createPlainProgressEmitter(io);
}
function isTuiCapableDemoIo(io: KloDemoIo): io is KloDemoIo & KloMemoryFlowTuiIo {
return (
io.stdin?.isTTY === true &&
io.stdout.isTTY === true &&
typeof io.stdin.setRawMode === 'function' &&
typeof io.stdout.write === 'function'
);
}
interface EffectiveDemoOutputModeOptions {
requireInput?: boolean;
}
function effectiveDemoOutputMode(
outputMode: KloDemoOutputMode,
io: KloDemoIo,
env: NodeJS.ProcessEnv,
options: EffectiveDemoOutputModeOptions = {},
): KloDemoOutputMode {
if (outputMode !== 'viz') {
return outputMode;
}
const fallback = resolveVizFallback(io, env, { requireInput: options.requireInput ?? false });
if (!fallback.shouldDegrade) {
return outputMode;
}
warnVizFallbackOnce(io, fallback);
return 'plain';
}
function initialFullDemoMemoryFlowInput(projectDir: string): MemoryFlowReplayInput {
return {
runId: DEMO_FULL_JOB_ID,
connectionId: DEMO_CONNECTION_ID,
adapter: DEMO_ADAPTER,
status: 'running',
sourceDir: `${projectDir}/raw-sources/${DEMO_CONNECTION_ID}/${DEMO_ADAPTER}`,
syncId: 'pending',
errors: [],
events: [],
plannedWorkUnits: [],
details: { actions: [], provenance: [], transcripts: [] },
};
}
async function ensureDemoProjectForCommand(projectDir: string): Promise<void> {
await ensureDemoProject({ projectDir, force: false }).catch((error) => {
if (error instanceof Error && error.message.includes('Demo project already exists')) {
return null;
}
throw error;
});
}
async function prepareProjectForDemoCommand(args: KloDemoArgs, io: KloDemoIo, deps: KloDemoDeps): Promise<string | null> {
if (args.command === 'init' || args.command === 'reset' || args.command === 'doctor') {
return args.projectDir;
}
const prompts = deps.prompts ?? createClackDemoPromptAdapter();
const decision = await chooseDemoProjectForInteractiveRun({
projectDir: args.projectDir,
inputMode: args.inputMode,
io,
prompts,
});
if (decision.action === 'cancel') {
return null;
}
if (decision.reset) {
await resetDemoProject({ projectDir: decision.projectDir, force: true });
}
return decision.projectDir;
}
async function runReplayDemo(
projectDir: string,
outputMode: KloDemoOutputMode,
inputMode: KloDemoArgs['inputMode'],
io: KloDemoIo,
deps: KloDemoDeps,
env: NodeJS.ProcessEnv = process.env,
): Promise<number> {
await ensureDemoProjectForCommand(projectDir);
await writeStoredReplay(await loadProjectDemoReplay(projectDir), outputMode, inputMode, io, deps, env);
return 0;
}
async function runSeededDemo(
projectDir: string,
outputMode: KloDemoOutputMode,
inputMode: KloDemoArgs['inputMode'],
io: KloDemoIo,
deps: KloDemoDeps,
env: NodeJS.ProcessEnv = process.env,
): Promise<number> {
const result = await runDemoSeeded({ projectDir });
const resolvedOutputMode = effectiveDemoOutputMode(outputMode, io, env, {
requireInput: inputMode !== 'disabled',
});
if (resolvedOutputMode === 'json') {
io.stdout.write(`${JSON.stringify({ replay: result.replay, inspect: result.inspect }, null, 2)}\n`);
return 0;
}
if (resolvedOutputMode === 'viz') {
await writeStoredReplay(result.replay, resolvedOutputMode, inputMode, io, deps, env);
} else {
writeReplay(result.replay, resolvedOutputMode, io);
io.stdout.write('\n');
io.stdout.write(formatSeededInspect(result.inspect));
}
return 0;
}
export async function runKloDemo(args: KloDemoArgs, io: KloDemoIo = process, deps: KloDemoDeps = {}): Promise<number> {
try {
if (args.command === 'init') {
const result = await ensureDemoProject({ projectDir: args.projectDir, force: args.force });
io.stdout.write(`Demo project: ${result.projectDir}\n`);
io.stdout.write(`Config: ${result.configPath}\n`);
io.stdout.write(`Database: ${result.databasePath}\n`);
io.stdout.write(`Replay: ${result.replayPath}\n`);
io.stdout.write('Next: klo setup demo --no-input\n');
io.stdout.write(' Runs the pre-seeded demo without calling the LLM.\n');
return 0;
}
if (args.command === 'reset') {
const result = await resetDemoProject({ projectDir: args.projectDir, force: args.force });
io.stdout.write(`Demo project reset: ${result.projectDir}\n`);
io.stdout.write(`Config: ${result.configPath}\n`);
io.stdout.write(`Database: ${result.databasePath}\n`);
io.stdout.write(`Replay: ${result.replayPath}\n`);
io.stdout.write('Next: klo setup demo --mode full\n');
io.stdout.write(' Runs the full AI-backed pass with your LLM provider.\n');
return 0;
}
const preparedProjectDir = await prepareProjectForDemoCommand(args, io, deps);
if (preparedProjectDir === null) {
return 1;
}
const env = deps.env ?? process.env;
if (args.command === 'scan') {
const { result } = await runDemoScan({ projectDir: preparedProjectDir });
io.stdout.write(formatDemoScanSummary(result.report));
return 0;
}
if (args.command === 'seeded' || (args.command === 'ingest' && args.mode === 'seeded')) {
return await runSeededDemo(preparedProjectDir, args.outputMode, args.inputMode, io, deps, env);
}
if (args.command === 'full' || (args.command === 'ingest' && args.mode === 'full')) {
const executeFullDemo = deps.runFullDemo ?? runDemoFull;
await ensureDemoProjectForCommand(preparedProjectDir);
const project = await loadKloProject({ projectDir: preparedProjectDir });
const credentialStatus = fullDemoCredentialStatus(project, env);
const credentialDecision = await resolveFullCredentialDecision({
needsAnthropicKey:
credentialStatus.status === 'missing-anthropic-key' &&
project.config.llm.provider.backend === 'anthropic' &&
!resolveKloConfigReference(project.config.llm.provider.anthropic?.api_key, env),
inputMode: args.inputMode,
io,
env,
prompts: deps.prompts ?? createClackDemoPromptAdapter(),
});
if (credentialDecision.action === 'cancel') {
return 1;
}
if (credentialDecision.action === 'run-mode') {
return credentialDecision.mode === 'seeded'
? await runSeededDemo(preparedProjectDir, args.outputMode, args.inputMode, io, deps, env)
: await runReplayDemo(preparedProjectDir, args.outputMode, args.inputMode, io, deps, env);
}
let liveSession: MemoryFlowTuiLiveSession | null = null;
let liveWasRendered = false;
const startLiveMemoryFlow = deps.startLiveMemoryFlow ?? startLiveMemoryFlowTui;
let fullOutputMode = effectiveDemoOutputMode(args.outputMode, io, env, {
requireInput: args.inputMode !== 'disabled',
});
const shouldUseLiveViz = fullOutputMode === 'viz' && args.inputMode !== 'disabled';
if (shouldUseLiveViz && isTuiCapableDemoIo(io)) {
liveSession = await startLiveMemoryFlow(initialFullDemoMemoryFlowInput(preparedProjectDir), io);
liveWasRendered = liveSession !== null;
} else if (shouldUseLiveViz) {
warnVizFallbackOnce(io, rendererUnavailableVizFallback());
fullOutputMode = 'plain';
}
const onMemoryFlowChange = pickMemoryFlowProgress(liveSession, fullOutputMode, io);
const result = await executeFullDemo({
projectDir: preparedProjectDir,
env: credentialDecision.env,
...(onMemoryFlowChange ? { onMemoryFlowChange } : {}),
});
await writeDemoReplay(preparedProjectDir, replayWithFullMetadata(result), { label: 'full' });
liveSession?.close();
writeFullDemo(result, fullOutputMode, io, { liveWasRendered, projectDir: preparedProjectDir });
if (fullOutputMode !== 'json' && !liveWasRendered) {
io.stdout.write(formatDemoInspect(await inspectDemoProject(preparedProjectDir)));
}
return 0;
}
if (args.command === 'inspect') {
const seededInspect = await inspectSeededProject(preparedProjectDir).catch(() => null);
if (seededInspect?.mode === 'seeded') {
if (args.outputMode === 'json') {
io.stdout.write(`${JSON.stringify(seededInspect, null, 2)}\n`);
} else {
io.stdout.write(formatSeededInspect(seededInspect));
}
return 0;
}
writeInspect(await inspectDemoProject(preparedProjectDir), args.outputMode, io);
return 0;
}
if (args.command === 'doctor') {
const { runKloDoctor } = await import('./doctor.js');
const executeDoctor = deps.runDoctor ?? runKloDoctor;
return await executeDoctor(
{
command: 'demo',
projectDir: args.projectDir,
outputMode: args.outputMode,
...(args.inputMode ? { inputMode: args.inputMode } : {}),
},
io,
);
}
return await runReplayDemo(preparedProjectDir, args.outputMode, args.inputMode, io, deps, env);
} catch (error) {
io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
return 1;
}
}

View file

@ -0,0 +1,670 @@
import { describe, expect, it, vi } from 'vitest';
import { runKloCli } from './index.js';
function makeIo() {
let stdout = '';
let stderr = '';
return {
io: {
stdout: {
write: (chunk: string) => {
stdout += chunk;
},
},
stderr: {
write: (chunk: string) => {
stderr += chunk;
},
},
},
stdout: () => stdout,
stderr: () => stderr,
};
}
describe('dev Commander tree', () => {
it('prints visible dev help with only supported low-level command groups', async () => {
const testIo = makeIo();
await expect(runKloCli(['dev', '--help'], testIo.io)).resolves.toBe(0);
expect(testIo.stdout()).toContain('Usage: klo dev [options] [command]');
for (const command of ['init', 'doctor', 'scan', 'ingest', 'mapping']) {
expect(testIo.stdout()).toContain(command);
}
for (const removed of [
'knowledge',
'model',
'replay',
'report',
'status',
'artifacts',
'config',
'tools',
'daemon',
]) {
expect(testIo.stdout()).not.toContain(`${removed} `);
}
expect(testIo.stderr()).toBe('');
});
it('keeps dev callable while hiding it from root command rows', async () => {
const testIo = makeIo();
await expect(runKloCli(['--help'], testIo.io)).resolves.toBe(0);
expect(testIo.stdout()).toContain('Advanced:');
expect(testIo.stdout()).toContain('klo dev');
expect(testIo.stdout()).not.toContain('dev Low-level diagnostics');
expect(testIo.stderr()).toBe('');
});
it('keeps project scaffolding under dev init', async () => {
const { mkdtemp, readFile, rm } = await import('node:fs/promises');
const { tmpdir } = await import('node:os');
const { join } = await import('node:path');
const tempDir = await mkdtemp(join(tmpdir(), 'klo-dev-init-'));
const projectDir = join(tempDir, 'warehouse');
const testIo = makeIo();
try {
await expect(runKloCli(['dev', 'init', projectDir, '--name', 'warehouse'], testIo.io)).resolves.toBe(0);
expect(testIo.stdout()).toContain(`Initialized KLO project at ${projectDir}`);
await expect(readFile(join(projectDir, 'klo.yaml'), 'utf-8')).resolves.toContain('project: warehouse');
expect(testIo.stderr()).toBe('');
} finally {
await rm(tempDir, { recursive: true, force: true });
}
});
it('uses global project-dir for dev init when the positional directory is omitted', async () => {
const { mkdtemp, rm } = await import('node:fs/promises');
const { tmpdir } = await import('node:os');
const { join } = await import('node:path');
const tempDir = await mkdtemp(join(tmpdir(), 'klo-dev-init-global-'));
const projectDir = join(tempDir, 'global-init');
const testIo = makeIo();
try {
await expect(
runKloCli(['--project-dir', projectDir, 'dev', 'init', '--name', 'global-init'], testIo.io),
).resolves.toBe(0);
expect(testIo.stdout()).toContain(`Initialized KLO project at ${projectDir}`);
expect(testIo.stderr()).toBe('');
} finally {
await rm(tempDir, { recursive: true, force: true });
}
});
it('rejects removed dev command groups', async () => {
for (const argv of [
['dev', 'knowledge', 'list'],
['dev', 'model', 'list'],
['dev', 'artifacts'],
]) {
const testIo = makeIo();
await expect(runKloCli(argv, testIo.io)).resolves.toBe(1);
expect(testIo.stderr()).toMatch(/unknown command|error:/);
}
});
it.each([
{
argv: ['dev', 'doctor', '--help'],
expected: ['Usage: klo dev doctor', '--json', '--no-input'],
},
{
argv: ['dev', 'scan', '--help'],
expected: [
'Usage: klo dev scan',
'--mode <mode>',
'structural',
'relationships',
'--dry-run',
'status',
'report',
'relationships',
'relationship-apply',
'relationship-feedback',
'relationship-calibration',
'relationship-thresholds',
],
},
{
argv: ['dev', 'scan', 'report', '--help'],
expected: ['Usage: klo dev scan report [options] <runId>', '<runId>', '--json'],
},
{
argv: ['dev', 'scan', 'relationships', '--help'],
expected: [
'Usage: klo dev scan relationships [options] <runId>',
'--status <status>',
'--limit <count>',
'--accept <candidateId>',
'--reject <candidateId>',
'--note <text>',
'--reviewer <name>',
'--json',
],
},
{
argv: ['dev', 'scan', 'relationship-apply', '--help'],
expected: [
'Usage: klo dev scan relationship-apply [options] <runId>',
'--all-accepted',
'--candidate <candidateId>',
'--dry-run',
],
},
{
argv: ['dev', 'scan', 'relationship-thresholds', '--help'],
expected: [
'Usage: klo dev scan relationship-thresholds [options]',
'--connection <connectionId>',
'--min-total-labels <count>',
'--min-accepted-labels <count>',
'--min-rejected-labels <count>',
'--json',
],
},
{
argv: ['dev', 'scan', 'relationship-feedback', '--help'],
expected: [
'Usage: klo dev scan relationship-feedback [options]',
'--connection <connectionId>',
'--decision <decision>',
'--json',
'--jsonl',
],
},
{
argv: ['dev', 'scan', 'relationship-calibration', '--help'],
expected: [
'Usage: klo dev scan relationship-calibration [options]',
'--connection <connectionId>',
'--decision <decision>',
'--accept-threshold <value>',
'--review-threshold <value>',
'--json',
],
},
{
argv: ['dev', 'ingest', 'run', '--help'],
expected: ['Usage: klo dev ingest run [options]', '--connection-id <connectionId>', '--adapter <adapter>'],
},
{
argv: ['dev', 'mapping', 'sync-state', 'set', '--help'],
expected: ['Usage: klo dev mapping sync-state set [options] <connectionId>', '--mode <mode>'],
},
])('prints generated nested help for $argv', async ({ argv, expected }) => {
const io = makeIo();
const doctor = vi.fn(async () => 0);
const ingest = vi.fn(async () => 0);
const scan = vi.fn(async () => 0);
await expect(runKloCli(argv, io.io, { doctor, ingest, scan })).resolves.toBe(0);
for (const text of expected) {
expect(io.stdout()).toContain(text);
}
expect(io.stderr()).toBe('');
expect(doctor).not.toHaveBeenCalled();
expect(ingest).not.toHaveBeenCalled();
expect(scan).not.toHaveBeenCalled();
});
it('dispatches dev scan through Commander with injected dependencies', async () => {
const scanIo = makeIo();
const scan = vi.fn(async () => 0);
await expect(
runKloCli(['dev', 'scan', 'warehouse', '--project-dir', '/tmp/project', '--dry-run'], scanIo.io, { scan }),
).resolves.toBe(0);
expect(scan).toHaveBeenCalledWith(
{
command: 'run',
projectDir: '/tmp/project',
connectionId: 'warehouse',
mode: 'structural',
detectRelationships: false,
dryRun: true,
databaseIntrospectionUrl: undefined,
},
scanIo.io,
);
expect(scanIo.stderr()).toBe('');
});
it('dispatches dev scan --mode relationships through Commander', async () => {
const io = makeIo();
const scan = vi.fn(async () => 0);
await expect(
runKloCli(['dev', 'scan', 'warehouse', '--project-dir', '/tmp/project', '--mode', 'relationships'], io.io, {
scan,
}),
).resolves.toBe(0);
expect(scan).toHaveBeenCalledWith(
{
command: 'run',
projectDir: '/tmp/project',
connectionId: 'warehouse',
mode: 'relationships',
detectRelationships: true,
dryRun: false,
databaseIntrospectionUrl: undefined,
},
io.io,
);
expect(io.stderr()).toBe('');
});
it.each(['--enrich', '--detect-relationships'])('rejects removed scan shorthand option %s', async (option) => {
const io = makeIo();
const scan = vi.fn(async () => 0);
await expect(runKloCli(['dev', 'scan', 'warehouse', option], io.io, { scan })).resolves.toBe(1);
expect(scan).not.toHaveBeenCalled();
expect(io.stderr()).toContain(`unknown option '${option}'`);
});
it('rejects dev scan without a connection id or subcommand', async () => {
const io = makeIo();
const scan = vi.fn(async () => 0);
await expect(runKloCli(['dev', 'scan', '--dry-run'], io.io, { scan })).resolves.toBe(1);
expect(scan).not.toHaveBeenCalled();
expect(io.stdout()).toContain('Usage: klo dev scan');
expect(io.stderr()).toContain('klo dev scan requires <connectionId> or a subcommand');
});
it('rejects invalid scan modes before dispatch', async () => {
const io = makeIo();
const scan = vi.fn(async () => 0);
await expect(runKloCli(['dev', 'scan', 'warehouse', '--mode', 'deep'], io.io, { scan })).resolves.toBe(1);
expect(scan).not.toHaveBeenCalled();
expect(io.stderr()).toContain("argument 'deep' is invalid");
expect(io.stderr()).toContain('Allowed choices are structural, enriched, relationships');
});
it('prints dev scan subcommand help with the canonical command name', async () => {
const io = makeIo();
const scan = vi.fn(async () => 0);
await expect(runKloCli(['dev', 'scan', 'report', '--help'], io.io, { scan })).resolves.toBe(0);
expect(io.stdout()).toContain('--project-dir is inherited from `klo dev scan`');
expect(io.stdout()).not.toContain('--project-dir is inherited from `klo scan`');
expect(scan).not.toHaveBeenCalled();
});
it('dispatches dev scan report in human and json modes', async () => {
const humanIo = makeIo();
const jsonIo = makeIo();
const scan = vi.fn(async () => 0);
await expect(
runKloCli(['dev', 'scan', 'report', 'scan-run-1', '--project-dir', '/tmp/project'], humanIo.io, { scan }),
).resolves.toBe(0);
await expect(
runKloCli(['dev', 'scan', 'report', 'scan-run-2', '--project-dir', '/tmp/project', '--json'], jsonIo.io, {
scan,
}),
).resolves.toBe(0);
expect(scan).toHaveBeenNthCalledWith(
1,
{ command: 'report', projectDir: '/tmp/project', runId: 'scan-run-1', json: false },
humanIo.io,
);
expect(scan).toHaveBeenNthCalledWith(
2,
{ command: 'report', projectDir: '/tmp/project', runId: 'scan-run-2', json: true },
jsonIo.io,
);
});
it('dispatches dev scan relationships with filters through Commander', async () => {
const io = makeIo();
const scan = vi.fn(async () => 0);
await expect(
runKloCli(
[
'dev',
'scan',
'relationships',
'scan-run-review',
'--project-dir',
'/tmp/project',
'--status',
'rejected',
'--limit',
'5',
'--json',
],
io.io,
{ scan },
),
).resolves.toBe(0);
expect(scan).toHaveBeenCalledWith(
{
command: 'relationships',
projectDir: '/tmp/project',
runId: 'scan-run-review',
status: 'rejected',
json: true,
limit: 5,
},
io.io,
);
expect(io.stderr()).toBe('');
});
it('dispatches dev scan relationship decision recording through Commander', async () => {
const io = makeIo();
const scan = vi.fn(async () => 0);
await expect(
runKloCli(
[
'dev',
'scan',
'relationships',
'scan-run-review',
'--project-dir',
'/tmp/project',
'--accept',
'orders:orders.customer_id->customers:customers.id',
'--reviewer',
'Andrey',
'--note',
'Looks right',
'--json',
],
io.io,
{ scan },
),
).resolves.toBe(0);
expect(scan).toHaveBeenCalledWith(
{
command: 'relationshipDecision',
projectDir: '/tmp/project',
runId: 'scan-run-review',
candidateId: 'orders:orders.customer_id->customers:customers.id',
decision: 'accepted',
reviewer: 'Andrey',
note: 'Looks right',
json: true,
},
io.io,
);
expect(io.stderr()).toBe('');
});
it.each(['--accept', '--reject'])('rejects empty relationship decision candidate ids for %s', async (option) => {
const io = makeIo();
const scan = vi.fn(async () => 0);
await expect(
runKloCli(['dev', 'scan', 'relationships', 'scan-run-review', option, ''], io.io, { scan }),
).resolves.toBe(1);
expect(scan).not.toHaveBeenCalled();
expect(io.stderr()).toContain('must not be empty');
});
it('rejects relationship feedback JSON and JSONL output together', async () => {
const io = makeIo();
const scan = vi.fn(async () => 0);
await expect(
runKloCli(['dev', 'scan', 'relationship-feedback', '--json', '--jsonl'], io.io, { scan }),
).resolves.toBe(1);
expect(scan).not.toHaveBeenCalled();
expect(io.stderr()).toMatch(/conflict|cannot be used/i);
});
it('dispatches relationship apply command args', async () => {
const io = makeIo();
const scan = vi.fn(async () => 0);
await expect(
runKloCli(
[
'dev',
'scan',
'relationship-apply',
'scan-run-a',
'--project-dir',
'/tmp/project',
'--candidate',
'orders:orders.customer_id->customers:customers.id',
'--dry-run',
'--json',
],
io.io,
{ scan },
),
).resolves.toBe(0);
expect(scan).toHaveBeenCalledWith(
{
command: 'relationshipApply',
projectDir: '/tmp/project',
runId: 'scan-run-a',
applyAllAccepted: false,
candidateIds: ['orders:orders.customer_id->customers:customers.id'],
dryRun: true,
json: true,
},
io.io,
);
});
it('dispatches scan relationship feedback command with filters and JSONL output', async () => {
const io = makeIo();
const scan = vi.fn(async () => 0);
await expect(
runKloCli(
[
'dev',
'scan',
'relationship-feedback',
'--project-dir',
'/tmp/project',
'--connection',
'warehouse',
'--decision',
'accepted',
'--jsonl',
],
io.io,
{ scan },
),
).resolves.toBe(0);
expect(scan).toHaveBeenCalledWith(
{
command: 'relationshipFeedback',
projectDir: '/tmp/project',
connectionId: 'warehouse',
decision: 'accepted',
json: false,
jsonl: true,
},
io.io,
);
});
it('dispatches scan relationship calibration command with thresholds', async () => {
const io = makeIo();
const scan = vi.fn(async () => 0);
await expect(
runKloCli(
[
'dev',
'scan',
'relationship-calibration',
'--project-dir',
'/tmp/project',
'--connection',
'warehouse',
'--decision',
'rejected',
'--accept-threshold',
'0.9',
'--review-threshold',
'0.5',
'--json',
],
io.io,
{ scan },
),
).resolves.toBe(0);
expect(scan).toHaveBeenCalledWith(
{
command: 'relationshipCalibration',
projectDir: '/tmp/project',
connectionId: 'warehouse',
decision: 'rejected',
acceptThreshold: 0.9,
reviewThreshold: 0.5,
json: true,
},
io.io,
);
});
it('dispatches relationship threshold advice command args', async () => {
const io = makeIo();
const scan = vi.fn(async () => 0);
await expect(
runKloCli(
[
'dev',
'scan',
'relationship-thresholds',
'--project-dir',
'/tmp/project',
'--connection',
'warehouse',
'--min-total-labels',
'12',
'--min-accepted-labels',
'4',
'--min-rejected-labels',
'3',
'--json',
],
io.io,
{ scan },
),
).resolves.toBe(0);
expect(scan).toHaveBeenCalledWith(
{
command: 'relationshipThresholds',
projectDir: '/tmp/project',
connectionId: 'warehouse',
minTotalLabels: 12,
minAcceptedLabels: 4,
minRejectedLabels: 3,
json: true,
},
io.io,
);
});
it('rejects invalid relationship calibration thresholds before dispatch', async () => {
const io = makeIo();
const scan = vi.fn(async () => 0);
await expect(
runKloCli(['dev', 'scan', 'relationship-calibration', '--accept-threshold', '1.5'], io.io, { scan }),
).resolves.toBe(1);
expect(scan).not.toHaveBeenCalled();
expect(io.stderr()).toContain('Allowed range is 0 through 1');
});
it('rejects relationship accept and reject options together before dispatch', async () => {
const io = makeIo();
const scan = vi.fn(async () => 0);
await expect(
runKloCli(
[
'dev',
'scan',
'relationships',
'scan-run-review',
'--accept',
'orders:orders.customer_id->customers:customers.id',
'--reject',
'orders:orders.customer_id->customers:customers.id',
],
io.io,
{ scan },
),
).resolves.toBe(1);
expect(scan).not.toHaveBeenCalled();
expect(io.stderr()).toMatch(/conflict|cannot be used/i);
});
it('dispatches dev ingest run through the low-level ingest Commander registration', async () => {
const io = makeIo();
const ingest = vi.fn(async () => 0);
await expect(
runKloCli(
[
'dev',
'ingest',
'run',
'--connection-id',
'warehouse',
'--adapter',
'metabase',
'--project-dir',
'/tmp/project',
'--json',
],
io.io,
{ ingest },
),
).resolves.toBe(0);
expect(ingest).toHaveBeenCalledWith(
{
command: 'run',
projectDir: '/tmp/project',
connectionId: 'warehouse',
adapter: 'metabase',
sourceDir: undefined,
databaseIntrospectionUrl: undefined,
outputMode: 'json',
},
io.io,
);
expect(io.stderr()).toBe('');
});
});

61
packages/cli/src/dev.ts Normal file
View file

@ -0,0 +1,61 @@
import { resolve } from 'node:path';
import type { Command } from '@commander-js/extra-typings';
import { type CommandWithGlobalOptions, type KloCliCommandContext, resolveCommandProjectDir } from './cli-program.js';
import { registerCompletionCommands } from './commands/completion-commands.js';
import { registerConnectionMappingCommands } from './commands/connection-commands.js';
import { registerDoctorCommands } from './commands/doctor-commands.js';
import { registerIngestCommands } from './commands/ingest-commands.js';
import { registerScanCommands } from './commands/scan-commands.js';
import { profileMark } from './startup-profile.js';
profileMark('module:dev');
export function registerDevCommands(program: Command, context: KloCliCommandContext): void {
const dev = program
.command('dev', { hidden: true })
.description('Low-level diagnostics, scans, adapter commands, and mapping tools')
.showHelpAfterError();
dev.hook('preAction', (_thisCommand, actionCommand) => {
context.writeDebug?.('dev', actionCommand);
});
dev.action(() => {
dev.outputHelp();
context.setExitCode(0);
});
dev
.command('init')
.description('Initialize a Git-backed KLO project directory for maintenance scripts')
.argument('[directory]', 'Project directory')
.option('--name <name>', 'Project name written to klo.yaml')
.option('--force', 'Rewrite klo.yaml and scaffold files in an existing project', false)
.action(
async (
projectDir: string | undefined,
commandOptions: { name?: string; force?: boolean },
command: CommandWithGlobalOptions,
) => {
context.setExitCode(
await context.runInit(
{
projectDir: projectDir ? resolve(projectDir) : resolveCommandProjectDir(command),
...(commandOptions.name ? { projectName: commandOptions.name } : {}),
force: commandOptions.force === true,
},
context.io,
),
);
},
);
registerDoctorCommands(dev, context);
registerScanCommands(dev, context);
registerIngestCommands(dev, context, {
runIngestWithProgress: async (ingestArgs, ingestIo, ingestDeps, defaultRunIngest) =>
await (ingestDeps.ingest ?? defaultRunIngest)(ingestArgs, ingestIo),
});
registerConnectionMappingCommands(dev, context);
registerCompletionCommands(dev, context, program);
}

View file

@ -0,0 +1,460 @@
import { mkdtemp, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import type { KloEmbeddingConfig, KloEmbeddingHealthCheckOptions, KloEmbeddingHealthCheckResult } from '@klo/llm';
import {
formatDoctorReport,
runKloDoctor,
runSetupDoctorChecks,
type DoctorCheck,
} from './doctor.js';
function makeIo() {
let stdout = '';
let stderr = '';
return {
io: {
stdout: {
write: (chunk: string) => {
stdout += chunk;
},
},
stderr: {
write: (chunk: string) => {
stderr += chunk;
},
},
},
stdout: () => stdout,
stderr: () => stderr,
};
}
type EmbeddingHealthCheck = (
config: KloEmbeddingConfig,
options?: KloEmbeddingHealthCheckOptions,
) => Promise<KloEmbeddingHealthCheckResult>;
async function writeProjectConfig(projectDir: string, embeddingLines: string[]): Promise<void> {
await writeFile(
join(projectDir, 'klo.yaml'),
[
'project: warehouse',
'connections:',
' warehouse:',
' driver: sqlite',
' path: ./warehouse.db',
'ingest:',
' adapters:',
' - live-database',
' embeddings:',
...embeddingLines.map((line) => ` ${line}`),
'',
].join('\n'),
'utf-8',
);
}
describe('formatDoctorReport', () => {
it('prints exact fixes for failing setup checks', () => {
const checks: DoctorCheck[] = [
{ id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0 ABI 127' },
{
id: 'native-sqlite',
label: 'Native SQLite',
status: 'fail',
detail: 'Cannot load better-sqlite3',
fix: 'Run: pnpm run native:rebuild',
},
];
expect(formatDoctorReport({ title: 'KLO setup doctor', checks })).toBe(
[
'KLO setup doctor',
'PASS Node 22+: v22.16.0 ABI 127',
'FAIL Native SQLite: Cannot load better-sqlite3',
' Fix: Run: pnpm run native:rebuild',
'',
].join('\n'),
);
});
});
describe('runSetupDoctorChecks', () => {
it('returns pass checks when injected commands and file checks succeed', async () => {
const checks = await runSetupDoctorChecks({
env: { PATH: '/bin' },
workspaceRoot: '/workspace/klo',
execText: async (command, args) => {
if (command === 'pnpm' && args[0] === '--version') return '10.28.0';
if (command === 'corepack' && args[0] === '--version') return '0.32.0';
if (command === 'uv' && args[0] === '--version') return 'uv 0.9.5';
if (command === process.execPath && args.includes('--version')) return '@klo/cli 0.0.0-private';
throw new Error(`${command} ${args.join(' ')}`);
},
pathExists: async () => true,
importBetterSqlite3: async () => ({ default: function Database() {} }),
});
expect(checks.map((check) => [check.id, check.status])).toEqual([
['node', 'pass'],
['pnpm', 'pass'],
['corepack', 'pass'],
['uv', 'pass'],
['native-sqlite', 'pass'],
['package-build', 'pass'],
['workspace-cli', 'pass'],
]);
});
it('returns exact fixes when setup checks fail', async () => {
const checks = await runSetupDoctorChecks({
env: {},
workspaceRoot: '/workspace/klo',
execText: async (command) => {
throw new Error(`${command} not found`);
},
pathExists: async () => false,
importBetterSqlite3: async () => {
throw new Error('Cannot find module better-sqlite3');
},
});
expect(checks).toContainEqual({
id: 'pnpm',
label: 'pnpm 10.20+',
status: 'fail',
detail: 'pnpm not found',
fix: 'Run: corepack enable && corepack prepare pnpm@10.28.0 --activate',
});
expect(checks).toContainEqual({
id: 'package-build',
label: 'TypeScript package build',
status: 'fail',
detail: 'Missing packages/cli/dist/bin.js',
fix: 'Run: pnpm run build',
});
});
it('treats missing corepack as a warning so setup doctor can still pass', async () => {
const checks = await runSetupDoctorChecks({
env: { PATH: '/bin' },
workspaceRoot: '/workspace/klo',
execText: async (command, args) => {
if (command === 'pnpm' && args[0] === '--version') return '10.28.0';
if (command === 'corepack' && args[0] === '--version') throw new Error('spawn corepack ENOENT');
if (command === 'uv' && args[0] === '--version') return 'uv 0.9.5';
if (command === process.execPath && args.includes('--version')) return '@klo/cli 0.0.0-private';
throw new Error(`${command} ${args.join(' ')}`);
},
pathExists: async () => true,
importBetterSqlite3: async () => ({ default: function Database() {} }),
});
const testIo = makeIo();
await expect(
runKloDoctor({ command: 'setup', outputMode: 'plain', inputMode: 'disabled' }, testIo.io, {
runSetupChecks: async () => checks,
}),
).resolves.toBe(0);
expect(checks).toContainEqual({
id: 'corepack',
label: 'Corepack',
status: 'warn',
detail: 'spawn corepack ENOENT',
fix: 'Run: corepack enable',
});
expect(testIo.stdout()).toContain('WARN Corepack: spawn corepack ENOENT');
expect(testIo.stderr()).toBe('');
});
});
describe('runKloDoctor', () => {
let tempDir: string;
beforeEach(async () => {
tempDir = await mkdtemp(join(tmpdir(), 'klo-doctor-'));
});
afterEach(async () => {
await rm(tempDir, { recursive: true, force: true });
});
it('prints setup report and exits nonzero when a check fails', async () => {
const testIo = makeIo();
await expect(
runKloDoctor(
{ command: 'setup', outputMode: 'plain', inputMode: 'disabled' },
testIo.io,
{
runSetupChecks: async () => [
{ id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0 ABI 127' },
{
id: 'package-build',
label: 'TypeScript package build',
status: 'fail',
detail: 'Missing packages/cli/dist/bin.js',
fix: 'Run: pnpm run build',
},
],
},
),
).resolves.toBe(1);
expect(testIo.stdout()).toContain('KLO setup doctor');
expect(testIo.stdout()).toContain('FAIL TypeScript package build: Missing packages/cli/dist/bin.js');
expect(testIo.stdout()).toContain('Fix: Run: pnpm run build');
expect(testIo.stderr()).toBe('');
});
it('prints JSON setup report', async () => {
const testIo = makeIo();
await expect(
runKloDoctor(
{ command: 'setup', outputMode: 'json', inputMode: 'disabled' },
testIo.io,
{
runSetupChecks: async () => [
{ id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0 ABI 127' },
],
},
),
).resolves.toBe(0);
expect(JSON.parse(testIo.stdout())).toEqual({
title: 'KLO setup doctor',
checks: [{ id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0 ABI 127' }],
});
});
it('runs project checks against a valid klo.yaml', async () => {
await writeFile(
join(tempDir, 'klo.yaml'),
[
'project: warehouse',
'connections:',
' warehouse:',
' driver: sqlite',
' path: ./warehouse.db',
'ingest:',
' adapters:',
' - live-database',
'',
].join('\n'),
'utf-8',
);
const testIo = makeIo();
await expect(
runKloDoctor(
{ command: 'project', projectDir: tempDir, outputMode: 'plain', inputMode: 'disabled' },
testIo.io,
{
runSetupChecks: async () => [
{ id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0 ABI 127' },
],
},
),
).resolves.toBe(0);
expect(testIo.stdout()).toContain('KLO project doctor');
expect(testIo.stdout()).toContain('PASS Project config: warehouse');
expect(testIo.stdout()).toContain('PASS Connections: 1 configured');
});
it('includes Postgres historic-SQL readiness in project doctor output', async () => {
await writeFile(
join(tempDir, 'klo.yaml'),
[
'project: warehouse',
'connections:',
' warehouse:',
' driver: postgres',
' url: env:WAREHOUSE_DATABASE_URL',
' readonly: true',
' historicSql:',
' enabled: true',
' dialect: postgres',
'ingest:',
' adapters:',
' - live-database',
' - historic-sql',
'',
].join('\n'),
'utf-8',
);
const testIo = makeIo();
const runHistoricSqlDoctorChecks = vi.fn(async () => [
{
id: 'historic-sql-postgres-warehouse',
label: 'Postgres Historic SQL (warehouse)',
status: 'warn' as const,
detail:
'pg_stat_statements ready (PostgreSQL 16.4) with warnings: pg_stat_statements.max is 1000; set it to at least 5000 to reduce query-template eviction churn',
fix: `Update the Postgres parameter group or config, then rerun \`klo dev doctor --project-dir ${tempDir}\``,
},
]);
await expect(
runKloDoctor(
{ command: 'project', projectDir: tempDir, outputMode: 'plain', inputMode: 'disabled' },
testIo.io,
{
runSetupChecks: async () => [
{ id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0 ABI 127' },
],
runHistoricSqlDoctorChecks,
},
),
).resolves.toBe(0);
expect(runHistoricSqlDoctorChecks).toHaveBeenCalledTimes(1);
expect(testIo.stdout()).toContain('WARN Postgres Historic SQL (warehouse): pg_stat_statements ready');
expect(testIo.stdout()).toContain('Fix: Update the Postgres parameter group or config');
});
it('warns when semantic-search embeddings are not configured', async () => {
await writeProjectConfig(tempDir, ['backend: deterministic', 'model: deterministic', 'dimensions: 8']);
const testIo = makeIo();
await expect(
runKloDoctor(
{ command: 'project', projectDir: tempDir, outputMode: 'plain', inputMode: 'disabled' },
testIo.io,
{
runSetupChecks: async () => [
{ id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0 ABI 127' },
],
},
),
).resolves.toBe(0);
expect(testIo.stdout()).toContain('WARN Semantic search embeddings: ingest.embeddings.backend is deterministic.');
expect(testIo.stdout()).toContain(
'Semantic lane will be skipped; lexical, dictionary, and token lanes remain available.',
);
expect(testIo.stdout()).toContain(
`Fix: Run: klo setup --project-dir ${tempDir} --no-input`,
);
});
it('probes configured semantic-search embeddings for project doctor', async () => {
await writeProjectConfig(tempDir, [
'backend: sentence-transformers',
'model: all-MiniLM-L6-v2',
'dimensions: 384',
'sentenceTransformers:',
' base_url: http://127.0.0.1:8765',
" pathPrefix: ''",
]);
const healthCheck = vi.fn<EmbeddingHealthCheck>(async () => ({ ok: true }));
const testIo = makeIo();
await expect(
runKloDoctor(
{ command: 'project', projectDir: tempDir, outputMode: 'plain', inputMode: 'disabled' },
testIo.io,
{
runSetupChecks: async () => [
{ id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0 ABI 127' },
],
embeddingHealthCheck: healthCheck,
embeddingProbeTimeoutMs: 1234,
},
),
).resolves.toBe(0);
expect(healthCheck).toHaveBeenCalledWith(
{
backend: 'sentence-transformers',
model: 'all-MiniLM-L6-v2',
dimensions: 384,
sentenceTransformers: { baseURL: 'http://127.0.0.1:8765', pathPrefix: '' },
},
{ text: 'KLO semantic search doctor probe', timeoutMs: 1234 },
);
expect(testIo.stdout()).toContain(
'PASS Semantic search embeddings: sentence-transformers/all-MiniLM-L6-v2 (384d) probe succeeded',
);
});
it('allows local sentence-transformers semantic-search probes enough time for cold start', async () => {
await writeProjectConfig(tempDir, [
'backend: sentence-transformers',
'model: all-MiniLM-L6-v2',
'dimensions: 384',
'sentenceTransformers:',
' base_url: http://127.0.0.1:8765',
" pathPrefix: ''",
]);
const healthCheck = vi.fn<EmbeddingHealthCheck>(async () => ({ ok: true }));
const testIo = makeIo();
await expect(
runKloDoctor(
{ command: 'project', projectDir: tempDir, outputMode: 'plain', inputMode: 'disabled' },
testIo.io,
{
runSetupChecks: async () => [
{ id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0 ABI 127' },
],
embeddingHealthCheck: healthCheck,
},
),
).resolves.toBe(0);
expect(healthCheck).toHaveBeenCalledWith(
expect.objectContaining({
backend: 'sentence-transformers',
model: 'all-MiniLM-L6-v2',
dimensions: 384,
}),
{ text: 'KLO semantic search doctor probe', timeoutMs: 120_000 },
);
});
it('reports unhealthy semantic-search embeddings as a warning in JSON output', async () => {
await writeProjectConfig(tempDir, [
'backend: sentence-transformers',
'model: all-MiniLM-L6-v2',
'dimensions: 384',
'sentenceTransformers:',
' base_url: http://127.0.0.1:8765',
" pathPrefix: ''",
]);
const healthCheck = vi.fn<EmbeddingHealthCheck>(async () => ({
ok: false,
message: 'connect ECONNREFUSED 127.0.0.1:8765',
}));
const testIo = makeIo();
await expect(
runKloDoctor(
{ command: 'project', projectDir: tempDir, outputMode: 'json', inputMode: 'disabled' },
testIo.io,
{
runSetupChecks: async () => [
{ id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0 ABI 127' },
],
embeddingHealthCheck: healthCheck,
},
),
).resolves.toBe(0);
const report = JSON.parse(testIo.stdout()) as {
checks: Array<{ id: string; label: string; status: string; detail: string; fix?: string }>;
};
expect(report.checks).toContainEqual({
id: 'semantic-search-embeddings',
label: 'Semantic search embeddings',
status: 'warn',
detail:
'sentence-transformers/all-MiniLM-L6-v2 (384d) probe failed: connect ECONNREFUSED 127.0.0.1:8765. Semantic lane will be skipped; lexical, dictionary, and token lanes remain available.',
fix: `Run: klo setup --project-dir ${tempDir} --no-input`,
});
});
});

488
packages/cli/src/doctor.ts Normal file
View file

@ -0,0 +1,488 @@
import { execFile } from 'node:child_process';
import { constants as fsConstants } from 'node:fs';
import { access } from 'node:fs/promises';
import { join, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import { promisify } from 'node:util';
import type { KloLocalProject, KloProjectEmbeddingConfig } from '@klo/context/project';
import type { KloEmbeddingConfig, KloEmbeddingHealthCheckOptions, KloEmbeddingHealthCheckResult } from '@klo/llm';
import type { HistoricSqlDoctorDeps } from './historic-sql-doctor.js';
const execFileAsync = promisify(execFile);
type DoctorStatus = 'pass' | 'warn' | 'fail';
type KloDoctorOutputMode = 'plain' | 'json';
type KloDoctorInputMode = 'auto' | 'disabled';
export interface DoctorCheck {
id: string;
label: string;
status: DoctorStatus;
detail: string;
fix?: string;
}
interface DoctorReport {
title: string;
checks: DoctorCheck[];
}
export type KloDoctorArgs =
| { command: 'setup'; outputMode: KloDoctorOutputMode; inputMode?: KloDoctorInputMode }
| { command: 'project'; projectDir: string; outputMode: KloDoctorOutputMode; inputMode?: KloDoctorInputMode }
| { command: 'demo'; projectDir: string; outputMode: KloDoctorOutputMode; inputMode?: KloDoctorInputMode };
interface KloDoctorIo {
stdout: { write(chunk: string): void };
stderr: { write(chunk: string): void };
}
interface SetupDoctorDeps {
env?: NodeJS.ProcessEnv;
workspaceRoot?: string;
execText?: (command: string, args: string[], options?: { cwd?: string; env?: NodeJS.ProcessEnv }) => Promise<string>;
pathExists?: (path: string) => Promise<boolean>;
importBetterSqlite3?: () => Promise<unknown>;
}
type EmbeddingHealthCheck = (
config: KloEmbeddingConfig,
options?: KloEmbeddingHealthCheckOptions,
) => Promise<KloEmbeddingHealthCheckResult>;
interface SemanticSearchDoctorDeps {
env?: NodeJS.ProcessEnv;
embeddingHealthCheck?: EmbeddingHealthCheck;
embeddingProbeTimeoutMs?: number;
}
interface KloDoctorDeps extends SemanticSearchDoctorDeps, HistoricSqlDoctorDeps {
runSetupChecks?: () => Promise<DoctorCheck[]>;
runHistoricSqlDoctorChecks?: (project: KloLocalProject, deps: HistoricSqlDoctorDeps) => Promise<DoctorCheck[]>;
}
function workspaceRootDir(): string {
return resolve(fileURLToPath(new URL('../../../', import.meta.url)));
}
async function defaultExecText(
command: string,
args: string[],
options: { cwd?: string; env?: NodeJS.ProcessEnv } = {},
): Promise<string> {
const result = await execFileAsync(command, args, {
cwd: options.cwd,
env: options.env,
encoding: 'utf8',
maxBuffer: 1024 * 1024,
});
return `${result.stdout}${result.stderr}`.trim();
}
async function defaultPathExists(path: string): Promise<boolean> {
try {
await access(path, fsConstants.F_OK);
return true;
} catch {
return false;
}
}
function failureMessage(error: unknown): string {
if (error instanceof Error && error.message.trim().length > 0) {
return error.message.trim().split('\n')[0] ?? error.message.trim();
}
return String(error);
}
function parseVersion(value: string): number[] {
const match = value.match(/(\d+)\.(\d+)\.(\d+)/);
if (!match) {
return [];
}
return [Number(match[1]), Number(match[2]), Number(match[3])];
}
function versionAtLeast(value: string, minimum: [number, number, number]): boolean {
const parsed = parseVersion(value);
if (parsed.length !== 3) {
return false;
}
for (let index = 0; index < minimum.length; index += 1) {
if (parsed[index] > minimum[index]) return true;
if (parsed[index] < minimum[index]) return false;
}
return true;
}
function check(status: DoctorStatus, id: string, label: string, detail: string, fix?: string): DoctorCheck {
return fix ? { id, label, status, detail, fix } : { id, label, status, detail };
}
const SEMANTIC_SEARCH_HEALTH_TEXT = 'KLO semantic search doctor probe';
const SEMANTIC_SEARCH_HEALTH_TIMEOUT_MS = 5_000;
const SEMANTIC_SEARCH_LOCAL_HEALTH_TIMEOUT_MS = 120_000;
function semanticEmbeddingSetupFix(projectDir: string, backend: KloProjectEmbeddingConfig['backend']): string {
if (backend === 'openai') {
return `Set OPENAI_API_KEY or rerun: klo setup --project-dir ${projectDir} --embedding-backend openai --no-input`;
}
return `Run: klo setup --project-dir ${projectDir} --no-input`;
}
function embeddingConfigLabel(config: KloProjectEmbeddingConfig | KloEmbeddingConfig): string {
const model = config.model?.trim() || 'model not configured';
return `${config.backend}/${model} (${config.dimensions}d)`;
}
function semanticLaneFallbackDetail(reason: string): string {
return `${reason}. Semantic lane will be skipped; lexical, dictionary, and token lanes remain available.`;
}
async function defaultEmbeddingHealthCheck(
config: KloEmbeddingConfig,
options?: KloEmbeddingHealthCheckOptions,
): Promise<KloEmbeddingHealthCheckResult> {
const { runKloEmbeddingHealthCheck } = await import('@klo/llm');
return runKloEmbeddingHealthCheck(config, options);
}
async function runSemanticSearchEmbeddingCheck(
config: KloProjectEmbeddingConfig,
projectDir: string,
deps: SemanticSearchDoctorDeps = {},
): Promise<DoctorCheck> {
if (config.backend === 'none' || config.backend === 'deterministic') {
return check(
'warn',
'semantic-search-embeddings',
'Semantic search embeddings',
semanticLaneFallbackDetail(`ingest.embeddings.backend is ${config.backend}`),
semanticEmbeddingSetupFix(projectDir, config.backend),
);
}
try {
const { resolveLocalKloEmbeddingConfig } = await import('@klo/context');
const resolved = resolveLocalKloEmbeddingConfig(config, deps.env ?? process.env);
if (!resolved) {
return check(
'warn',
'semantic-search-embeddings',
'Semantic search embeddings',
semanticLaneFallbackDetail(`No runtime embedding config resolved for ${embeddingConfigLabel(config)}`),
semanticEmbeddingSetupFix(projectDir, config.backend),
);
}
const healthCheck = deps.embeddingHealthCheck ?? defaultEmbeddingHealthCheck;
const timeoutMs =
deps.embeddingProbeTimeoutMs ??
(resolved.backend === 'sentence-transformers'
? SEMANTIC_SEARCH_LOCAL_HEALTH_TIMEOUT_MS
: SEMANTIC_SEARCH_HEALTH_TIMEOUT_MS);
const health = await healthCheck(resolved, {
text: SEMANTIC_SEARCH_HEALTH_TEXT,
timeoutMs,
});
if (health.ok) {
return check(
'pass',
'semantic-search-embeddings',
'Semantic search embeddings',
`${embeddingConfigLabel(resolved)} probe succeeded`,
);
}
return check(
'warn',
'semantic-search-embeddings',
'Semantic search embeddings',
semanticLaneFallbackDetail(`${embeddingConfigLabel(resolved)} probe failed: ${health.message}`),
semanticEmbeddingSetupFix(projectDir, config.backend),
);
} catch (error) {
return check(
'warn',
'semantic-search-embeddings',
'Semantic search embeddings',
semanticLaneFallbackDetail(`${embeddingConfigLabel(config)} probe failed: ${failureMessage(error)}`),
semanticEmbeddingSetupFix(projectDir, config.backend),
);
}
}
export async function runSetupDoctorChecks(deps: SetupDoctorDeps = {}): Promise<DoctorCheck[]> {
const env = deps.env ?? process.env;
const root = deps.workspaceRoot ?? workspaceRootDir();
const execText = deps.execText ?? defaultExecText;
const pathExists = deps.pathExists ?? defaultPathExists;
const importBetterSqlite3 = deps.importBetterSqlite3 ?? (() => import('better-sqlite3'));
const checks: DoctorCheck[] = [];
const nodeDetail = `${process.version} ABI ${process.versions.modules}`;
checks.push(
versionAtLeast(process.version, [22, 0, 0])
? check('pass', 'node', 'Node 22+', nodeDetail)
: check('fail', 'node', 'Node 22+', nodeDetail, 'Install Node 22 or newer, then rerun `pnpm run setup:dev`'),
);
try {
const pnpmVersion = await execText('pnpm', ['--version'], { cwd: root, env });
checks.push(
versionAtLeast(pnpmVersion, [10, 20, 0])
? check('pass', 'pnpm', 'pnpm 10.20+', pnpmVersion)
: check(
'fail',
'pnpm',
'pnpm 10.20+',
pnpmVersion,
'Run: corepack enable && corepack prepare pnpm@10.28.0 --activate',
),
);
} catch (error) {
checks.push(
check(
'fail',
'pnpm',
'pnpm 10.20+',
failureMessage(error),
'Run: corepack enable && corepack prepare pnpm@10.28.0 --activate',
),
);
}
try {
const corepackVersion = await execText('corepack', ['--version'], { cwd: root, env });
checks.push(check('pass', 'corepack', 'Corepack', corepackVersion));
} catch (error) {
checks.push(check('warn', 'corepack', 'Corepack', failureMessage(error), 'Run: corepack enable'));
}
try {
const uvVersion = await execText('uv', ['--version'], { cwd: root, env });
checks.push(check('pass', 'uv', 'uv', uvVersion));
} catch (error) {
checks.push(check('fail', 'uv', 'uv', failureMessage(error), 'Install uv, then rerun `pnpm run setup:dev`'));
}
try {
await importBetterSqlite3();
checks.push(check('pass', 'native-sqlite', 'Native SQLite', 'better-sqlite3 loaded'));
} catch (error) {
checks.push(
check('fail', 'native-sqlite', 'Native SQLite', failureMessage(error), 'Run: pnpm run native:rebuild'),
);
}
const cliBin = join(root, 'packages/cli/dist/bin.js');
if (await pathExists(cliBin)) {
checks.push(check('pass', 'package-build', 'TypeScript package build', 'packages/cli/dist/bin.js exists'));
} else {
checks.push(
check(
'fail',
'package-build',
'TypeScript package build',
'Missing packages/cli/dist/bin.js',
'Run: pnpm run build',
),
);
}
try {
const output = await execText(process.execPath, [cliBin, '--version'], { cwd: root, env });
checks.push(check('pass', 'workspace-cli', 'Workspace-local CLI', output));
} catch (error) {
checks.push(
check(
'fail',
'workspace-cli',
'Workspace-local CLI',
failureMessage(error),
'Run: pnpm run build && pnpm run klo -- --version',
),
);
}
return checks;
}
async function runProjectChecks(projectDir: string, deps: KloDoctorDeps = {}): Promise<DoctorCheck[]> {
const { loadKloProject } = await import('@klo/context/project');
const checks: DoctorCheck[] = [];
try {
const project = await loadKloProject({ projectDir });
checks.push(check('pass', 'project-config', 'Project config', project.config.project));
const connectionCount = Object.keys(project.config.connections).length;
checks.push(
connectionCount > 0
? check('pass', 'connections', 'Connections', `${connectionCount} configured`)
: check(
'warn',
'connections',
'Connections',
'0 configured',
'Add a connection to klo.yaml or run `klo setup demo init`',
),
);
checks.push(check('pass', 'storage', 'Storage', `${project.config.storage.state}/${project.config.storage.search}`));
checks.push(check('pass', 'llm-provider', 'LLM provider', project.config.llm.provider.backend));
checks.push(await runSemanticSearchEmbeddingCheck(project.config.ingest.embeddings, projectDir, deps));
const runHistoricSqlDoctorChecks =
deps.runHistoricSqlDoctorChecks ?? (await import('./historic-sql-doctor.js')).runPostgresHistoricSqlDoctorChecks;
checks.push(...(await runHistoricSqlDoctorChecks(project, deps)));
} catch (error) {
checks.push(
check(
'fail',
'project-config',
'Project config',
failureMessage(error),
`Run: klo init ${projectDir} --name <project-name>`,
),
);
}
return checks;
}
async function runDemoProjectChecks(projectDir: string, deps: KloDoctorDeps = {}): Promise<DoctorCheck[]> {
const env = deps.env ?? process.env;
const { DEMO_CONNECTION_ID, DEMO_REPLAY_FILE } = await import('./demo-assets.js');
const { loadKloProject } = await import('@klo/context/project');
const checks: DoctorCheck[] = [];
const requiredPaths = [
['demo-config', 'Demo config', 'klo.yaml'],
['demo-database', 'Demo dataset', 'demo.db'],
['demo-state', 'Demo state database', 'state.sqlite'],
['demo-replay', 'Demo replay', join('replays', DEMO_REPLAY_FILE)],
['demo-raw-sources', 'Demo raw sources directory', 'raw-sources'],
['demo-semantic-layer', 'Demo semantic-layer directory', 'semantic-layer'],
['demo-knowledge', 'Demo knowledge directory', 'knowledge'],
] as const;
for (const [id, label, relativePath] of requiredPaths) {
const absolutePath = join(projectDir, relativePath);
checks.push(
(await defaultPathExists(absolutePath))
? check('pass', id, label, relativePath)
: check(
'fail',
id,
label,
`Missing ${relativePath}`,
`Run: klo setup demo init --project-dir ${projectDir} --force --no-input`,
),
);
}
try {
const project = await loadKloProject({ projectDir });
const connection = project.config.connections[DEMO_CONNECTION_ID];
checks.push(
connection?.driver === 'sqlite'
? check('pass', 'demo-connection', 'Demo connection', `${DEMO_CONNECTION_ID} uses sqlite`)
: check(
'fail',
'demo-connection',
'Demo connection',
`${DEMO_CONNECTION_ID} is missing or is not sqlite`,
`Run: klo setup demo init --project-dir ${projectDir} --force --no-input`,
),
);
const provider = project.config.llm.provider.backend;
checks.push(
provider === 'anthropic' || provider === 'vertex' || provider === 'gateway'
? check('pass', 'demo-llm-provider', 'Demo LLM provider', provider)
: check(
'fail',
'demo-llm-provider',
'Demo LLM provider',
provider,
`Run: klo setup demo init --project-dir ${projectDir} --force --no-input`,
),
);
if (provider === 'anthropic' && !env.ANTHROPIC_API_KEY) {
checks.push(
check(
'warn',
'anthropic-credentials',
'Anthropic credentials',
'ANTHROPIC_API_KEY is not set',
'Export ANTHROPIC_API_KEY to run `klo setup demo --mode full --no-input`',
),
);
} else {
checks.push(check('pass', 'anthropic-credentials', 'Anthropic credentials', 'Configured for current provider'));
}
checks.push(await runSemanticSearchEmbeddingCheck(project.config.ingest.embeddings, projectDir, deps));
const runHistoricSqlDoctorChecks =
deps.runHistoricSqlDoctorChecks ?? (await import('./historic-sql-doctor.js')).runPostgresHistoricSqlDoctorChecks;
checks.push(...(await runHistoricSqlDoctorChecks(project, deps)));
} catch (error) {
checks.push(
check(
'fail',
'demo-config-parse',
'Demo config parse',
failureMessage(error),
`Run: klo setup demo init --project-dir ${projectDir} --force --no-input`,
),
);
}
return checks;
}
export function formatDoctorReport(report: DoctorReport): string {
const lines = [report.title];
for (const item of report.checks) {
lines.push(`${item.status.toUpperCase()} ${item.label}: ${item.detail}`);
if (item.fix) {
lines.push(` Fix: ${item.fix}`);
}
}
lines.push('');
return lines.join('\n');
}
function hasFailures(report: DoctorReport): boolean {
return report.checks.some((item) => item.status === 'fail');
}
function writeReport(report: DoctorReport, outputMode: KloDoctorOutputMode, io: KloDoctorIo): void {
if (outputMode === 'json') {
io.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
return;
}
io.stdout.write(formatDoctorReport(report));
}
export async function runKloDoctor(
args: KloDoctorArgs,
io: KloDoctorIo = process,
deps: KloDoctorDeps = {},
): Promise<number> {
try {
const runSetupChecks = deps.runSetupChecks ?? (() => runSetupDoctorChecks());
const setupChecks = await runSetupChecks();
const report: DoctorReport =
args.command === 'setup'
? { title: 'KLO setup doctor', checks: setupChecks }
: args.command === 'demo'
? {
title: 'KLO demo doctor',
checks: [...setupChecks, ...(await runDemoProjectChecks(args.projectDir, deps))],
}
: {
title: 'KLO project doctor',
checks: [...setupChecks, ...(await runProjectChecks(args.projectDir, deps))],
};
writeReport(report, args.outputMode, io);
return hasFailures(report) ? 1 : 0;
} catch (error) {
io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
return 1;
}
}

View file

@ -0,0 +1,252 @@
import { execFile } from 'node:child_process';
import { cp, mkdtemp, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join, resolve } from 'node:path';
import { promisify } from 'node:util';
import { Client } from '@modelcontextprotocol/sdk/client/index.js';
import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js';
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
const execFileAsync = promisify(execFile);
const CLI_BIN = resolve(process.cwd(), 'dist/bin.js');
const EXAMPLE_DIR = resolve(process.cwd(), '../../examples/local-warehouse');
interface CliResult {
code: number;
stdout: string;
stderr: string;
}
interface ExecFailure extends Error {
code?: number;
stdout?: string;
stderr?: string;
}
function isExecFailure(error: unknown): error is ExecFailure {
return error instanceof Error && ('stdout' in error || 'stderr' in error || 'code' in error);
}
async function runBuiltCli(args: string[]): Promise<CliResult> {
try {
const result = await execFileAsync(process.execPath, [CLI_BIN, ...args], {
encoding: 'utf8',
timeout: 20_000,
});
return {
code: 0,
stdout: result.stdout,
stderr: result.stderr,
};
} catch (error) {
if (!isExecFailure(error)) {
throw error;
}
return {
code: typeof error.code === 'number' ? error.code : 1,
stdout: error.stdout ?? '',
stderr: error.stderr ?? error.message,
};
}
}
function structuredContent<T extends object>(result: unknown): T {
const content = (result as { structuredContent?: unknown }).structuredContent;
expect(content).toBeDefined();
return content as T;
}
function parseJsonOutput<T>(stdout: string): T {
return JSON.parse(stdout) as T;
}
async function copyExampleProject(tempDir: string): Promise<string> {
const projectDir = join(tempDir, 'local-warehouse');
await cp(EXAMPLE_DIR, projectDir, { recursive: true });
return projectDir;
}
describe('standalone local warehouse example', () => {
let tempDir: string;
beforeEach(async () => {
tempDir = await mkdtemp(join(tmpdir(), 'klo-example-smoke-'));
});
afterEach(async () => {
await rm(tempDir, { recursive: true, force: true });
});
it('runs local CLI commands against the copied example project', async () => {
const projectDir = await copyExampleProject(tempDir);
const sourceDir = join(projectDir, 'source');
const knowledgeList = await runBuiltCli(['agent', 'wiki', 'search', 'revenue', '--json', '--project-dir', projectDir]);
expect(knowledgeList).toMatchObject({ code: 0, stderr: '' });
expect(parseJsonOutput<{ results: Array<{ key: string; summary: string }> }>(knowledgeList.stdout).results).toContainEqual(
expect.objectContaining({ key: 'revenue', summary: 'Paid order value after refunds' }),
);
const knowledgeRead = await runBuiltCli(['agent', 'wiki', 'read', 'revenue', '--json', '--project-dir', projectDir]);
expect(knowledgeRead).toMatchObject({ code: 0, stderr: '' });
expect(parseJsonOutput<{ content: string }>(knowledgeRead.stdout).content).toContain(
'Revenue is paid order amount after refund adjustments.',
);
const slList = await runBuiltCli(['agent', 'sl', 'list', '--json', '--project-dir', projectDir, '--connection-id', 'warehouse']);
expect(slList).toMatchObject({ code: 0, stderr: '' });
expect(parseJsonOutput<{ sources: Array<{ connectionId: string; name: string; columnCount: number }> }>(slList.stdout).sources).toContainEqual(
expect.objectContaining({ connectionId: 'warehouse', name: 'orders', columnCount: 3 }),
);
const slRead = await runBuiltCli([
'agent',
'sl',
'read',
'orders',
'--json',
'--connection-id',
'warehouse',
'--project-dir',
projectDir,
]);
expect(slRead).toMatchObject({ code: 0, stderr: '' });
expect(parseJsonOutput<{ yaml: string }>(slRead.stdout).yaml).toContain('name: orders');
const ingest = await runBuiltCli([
'dev',
'ingest',
'run',
'--project-dir',
projectDir,
'--connection-id',
'warehouse',
'--adapter',
'fake',
'--source-dir',
sourceDir,
]);
expect(ingest).toMatchObject({ code: 1, stdout: '' });
expect(ingest.stderr).toContain(
'klo dev ingest run requires llm.provider.backend: anthropic, vertex, or gateway, or an injected agentRunner',
);
}, 30_000);
it('serves local wiki and semantic-layer MCP tools against the copied example project', async () => {
const projectDir = await copyExampleProject(tempDir);
const client = new Client({ name: 'klo-example-client', version: '0.0.0' });
const transport = new StdioClientTransport({
command: process.execPath,
args: [CLI_BIN, 'serve', '--mcp', 'stdio', '--project-dir', projectDir, '--user-id', 'example-user'],
stderr: 'pipe',
});
try {
await client.connect(transport);
const knowledgeSearch = structuredContent<{
results: Array<{ key: string; summary: string; score: number }>;
totalFound: number;
}>(
await client.callTool({
name: 'knowledge_search',
arguments: { query: 'refund', limit: 5 },
}),
);
expect(knowledgeSearch.totalFound).toBe(1);
expect(knowledgeSearch.results[0]).toMatchObject({
key: 'revenue',
summary: 'Paid order value after refunds',
});
const knowledgeRead = structuredContent<{ key: string; summary: string; content: string; scope: string }>(
await client.callTool({ name: 'knowledge_read', arguments: { key: 'revenue' } }),
);
expect(knowledgeRead).toMatchObject({
key: 'revenue',
summary: 'Paid order value after refunds',
scope: 'GLOBAL',
});
expect(knowledgeRead.content).toContain('Revenue is paid order amount after refund adjustments.');
const knowledgeWrite = structuredContent<{ success: boolean; key: string; action: string }>(
await client.callTool({
name: 'knowledge_write',
arguments: {
key: 'gross_margin',
summary: 'Revenue after direct costs',
content: 'Gross margin subtracts direct order costs from revenue.',
tags: ['finance'],
sl_refs: ['warehouse.orders'],
},
}),
);
expect(knowledgeWrite).toEqual({ success: true, key: 'gross_margin', action: 'created' });
const slList = structuredContent<{
sources: Array<{
connectionId: string;
name: string;
description?: string;
columnCount: number;
measureCount: number;
joinCount: number;
}>;
totalSources: number;
}>(await client.callTool({ name: 'sl_list_sources', arguments: { connectionId: 'warehouse' } }));
expect(slList.totalSources).toBe(1);
expect(slList.sources[0]).toMatchObject({
connectionId: 'warehouse',
name: 'orders',
description: 'Orders placed through the storefront.',
columnCount: 3,
measureCount: 2,
joinCount: 0,
});
const slRead = structuredContent<{ sourceName: string; yaml: string }>(
await client.callTool({
name: 'sl_read_source',
arguments: { connectionId: 'warehouse', sourceName: 'orders' },
}),
);
expect(slRead.sourceName).toBe('orders');
expect(slRead.yaml).toContain('name: orders');
expect(slRead.yaml).toContain('total_revenue');
const slWrite = structuredContent<{ success: boolean; sourceName: string }>(
await client.callTool({
name: 'sl_write_source',
arguments: {
connectionId: 'warehouse',
sourceName: 'customers',
source: {
name: 'customers',
table: 'public.customers',
grain: ['id'],
columns: [{ name: 'id', type: 'number' }],
joins: [],
measures: [],
},
},
}),
);
expect(slWrite).toMatchObject({ success: true, sourceName: 'customers' });
const slValidate = structuredContent<{ success: boolean; errors: string[]; warnings: string[] }>(
await client.callTool({
name: 'sl_validate',
arguments: { connectionId: 'warehouse', names: ['orders', 'customers'] },
}),
);
expect(slValidate.success).toBe(true);
expect(slValidate.errors).toEqual([]);
expect(slValidate.warnings).toContain(
'Local stdio validation checks YAML shape only; Python semantic validation is not configured.',
);
} finally {
await client.close();
}
}, 30_000);
});

View file

@ -0,0 +1,173 @@
import { buildDefaultKloProjectConfig, type KloProjectConnectionConfig } from '@klo/context/project';
import { HistoricSqlExtensionMissingError } from '@klo/context/ingest';
import { describe, expect, it, vi } from 'vitest';
import {
runPostgresHistoricSqlDoctorChecks,
type HistoricSqlDoctorProject,
type PostgresHistoricSqlDoctorProbe,
} from './historic-sql-doctor.js';
function projectWithConnections(connections: Record<string, KloProjectConnectionConfig>): HistoricSqlDoctorProject {
return {
projectDir: '/tmp/klo-project',
config: {
...buildDefaultKloProjectConfig('warehouse'),
connections,
ingest: {
...buildDefaultKloProjectConfig('warehouse').ingest,
adapters: ['live-database', 'historic-sql'],
},
},
};
}
describe('runPostgresHistoricSqlDoctorChecks', () => {
it('passes when no Postgres historic-SQL connections are enabled', async () => {
const checks = await runPostgresHistoricSqlDoctorChecks(
projectWithConnections({
warehouse: { driver: 'sqlite', path: './warehouse.db', readonly: true },
}),
{
postgresHistoricSqlProbe: vi.fn<PostgresHistoricSqlDoctorProbe>(),
},
);
expect(checks).toEqual([
{
id: 'historic-sql-postgres',
label: 'Postgres Historic SQL',
status: 'pass',
detail: 'No enabled Postgres historic-SQL connections',
},
]);
});
it('passes when the PGSS probe succeeds without warnings', async () => {
const probe = vi.fn<PostgresHistoricSqlDoctorProbe>(async () => ({
pgServerVersion: 'PostgreSQL 16.4',
warnings: [],
}));
const checks = await runPostgresHistoricSqlDoctorChecks(
projectWithConnections({
warehouse: {
driver: 'postgres',
url: 'env:WAREHOUSE_DATABASE_URL',
readonly: true,
historicSql: { enabled: true, dialect: 'postgres' },
},
}),
{ postgresHistoricSqlProbe: probe },
);
expect(probe).toHaveBeenCalledWith({
projectDir: '/tmp/klo-project',
connectionId: 'warehouse',
connection: {
driver: 'postgres',
url: 'env:WAREHOUSE_DATABASE_URL',
readonly: true,
historicSql: { enabled: true, dialect: 'postgres' },
},
env: process.env,
});
expect(checks).toEqual([
{
id: 'historic-sql-postgres-warehouse',
label: 'Postgres Historic SQL (warehouse)',
status: 'pass',
detail: 'pg_stat_statements ready (PostgreSQL 16.4)',
},
]);
});
it('warns when the PGSS probe succeeds with operational warnings', async () => {
const checks = await runPostgresHistoricSqlDoctorChecks(
projectWithConnections({
warehouse: {
driver: 'postgres',
url: 'env:WAREHOUSE_DATABASE_URL',
readonly: true,
historicSql: { enabled: true, dialect: 'postgres' },
},
}),
{
postgresHistoricSqlProbe: async () => ({
pgServerVersion: 'PostgreSQL 16.4',
warnings: [
'pg_stat_statements.max is 1000; set it to at least 5000 to reduce query-template eviction churn',
],
}),
},
);
expect(checks).toEqual([
{
id: 'historic-sql-postgres-warehouse',
label: 'Postgres Historic SQL (warehouse)',
status: 'warn',
detail:
'pg_stat_statements ready (PostgreSQL 16.4) with warnings: pg_stat_statements.max is 1000; set it to at least 5000 to reduce query-template eviction churn',
fix: 'Update the Postgres parameter group or config, then rerun `klo dev doctor --project-dir /tmp/klo-project`',
},
]);
});
it('fails when a connection has postgres historic SQL but is not a Postgres driver', async () => {
const checks = await runPostgresHistoricSqlDoctorChecks(
projectWithConnections({
warehouse: {
driver: 'mysql',
url: 'env:WAREHOUSE_DATABASE_URL',
readonly: true,
historicSql: { enabled: true, dialect: 'postgres' },
},
}),
{
postgresHistoricSqlProbe: vi.fn<PostgresHistoricSqlDoctorProbe>(),
},
);
expect(checks).toEqual([
{
id: 'historic-sql-postgres-warehouse',
label: 'Postgres Historic SQL (warehouse)',
status: 'fail',
detail: 'connections.warehouse.historicSql.dialect is postgres but driver is mysql',
fix: 'Set connections.warehouse.driver to postgres or disable historicSql for this connection',
},
]);
});
it('maps PGSS capability errors to actionable failures', async () => {
const checks = await runPostgresHistoricSqlDoctorChecks(
projectWithConnections({
warehouse: {
driver: 'postgres',
url: 'env:WAREHOUSE_DATABASE_URL',
readonly: true,
historicSql: { enabled: true, dialect: 'postgres' },
},
}),
{
postgresHistoricSqlProbe: async () => {
throw new HistoricSqlExtensionMissingError({
dialect: 'postgres',
message: 'pg_stat_statements extension is not installed in the connection database.',
remediation: 'Run CREATE EXTENSION pg_stat_statements; against the connection database.',
});
},
},
);
expect(checks).toEqual([
{
id: 'historic-sql-postgres-warehouse',
label: 'Postgres Historic SQL (warehouse)',
status: 'fail',
detail: 'pg_stat_statements extension is not installed in the connection database.',
fix: 'Run CREATE EXTENSION pg_stat_statements; against the connection database.',
},
]);
});
});

View file

@ -0,0 +1,160 @@
import type { KloProjectConfig, KloProjectConnectionConfig } from '@klo/context/project';
import type { DoctorCheck } from './doctor.js';
export interface HistoricSqlDoctorProject {
projectDir: string;
config: Pick<KloProjectConfig, 'connections' | 'ingest'>;
}
export interface PostgresHistoricSqlDoctorProbeInput {
projectDir: string;
connectionId: string;
connection: KloProjectConnectionConfig;
env: NodeJS.ProcessEnv;
}
export interface PostgresHistoricSqlDoctorProbeResult {
pgServerVersion: string;
warnings: string[];
}
export type PostgresHistoricSqlDoctorProbe = (
input: PostgresHistoricSqlDoctorProbeInput,
) => Promise<PostgresHistoricSqlDoctorProbeResult>;
export interface HistoricSqlDoctorDeps {
env?: NodeJS.ProcessEnv;
postgresHistoricSqlProbe?: PostgresHistoricSqlDoctorProbe;
}
function check(status: DoctorCheck['status'], id: string, label: string, detail: string, fix?: string): DoctorCheck {
return fix ? { id, label, status, detail, fix } : { id, label, status, detail };
}
function historicSqlRecord(connection: KloProjectConnectionConfig): Record<string, unknown> | null {
const historicSql = connection.historicSql;
return historicSql && typeof historicSql === 'object' && !Array.isArray(historicSql)
? (historicSql as Record<string, unknown>)
: null;
}
function isEnabledPostgresHistoricSql(connection: KloProjectConnectionConfig): boolean {
const historicSql = historicSqlRecord(connection);
return historicSql?.enabled === true && historicSql.dialect === 'postgres';
}
function isPostgresDriver(connection: KloProjectConnectionConfig): boolean {
const driver = String(connection.driver ?? '').toLowerCase();
return driver === 'postgres' || driver === 'postgresql';
}
function checkId(connectionId: string): string {
return `historic-sql-postgres-${connectionId.replace(/[^a-z0-9_-]+/gi, '-')}`;
}
function capabilityFailureFix(error: unknown, connectionId: string, projectDir: string): string {
if (error instanceof Error && error.name === 'HistoricSqlExtensionMissingError' && 'remediation' in error) {
return String(error.remediation);
}
if (error instanceof Error && error.name === 'HistoricSqlGrantsMissingError' && 'remediation' in error) {
return String(error.remediation);
}
if (error instanceof Error && error.name === 'HistoricSqlVersionUnsupportedError') {
return 'Use PostgreSQL 14 or newer, or disable historicSql for this connection';
}
return `Fix connections.${connectionId} Postgres settings, then rerun \`klo dev doctor --project-dir ${projectDir}\``;
}
function failureDetail(error: unknown): string {
if (error instanceof Error && error.message.trim().length > 0) {
return error.message.trim().split('\n')[0] ?? error.message.trim();
}
return String(error);
}
async function defaultPostgresHistoricSqlProbe(
input: PostgresHistoricSqlDoctorProbeInput,
): Promise<PostgresHistoricSqlDoctorProbeResult> {
const [{ PostgresPgssQueryHistoryReader }, { KloPostgresHistoricSqlQueryClient, isKloPostgresConnectionConfig }] =
await Promise.all([import('@klo/context/ingest'), import('@klo/connector-postgres')]);
if (!isKloPostgresConnectionConfig(input.connection)) {
throw new Error(`Native PostgreSQL connector cannot run driver "${input.connection.driver ?? 'unknown'}"`);
}
const client = new KloPostgresHistoricSqlQueryClient({
connectionId: input.connectionId,
connection: input.connection,
env: input.env,
});
try {
return await new PostgresPgssQueryHistoryReader().probe(client);
} finally {
await client.cleanup();
}
}
export async function runPostgresHistoricSqlDoctorChecks(
project: HistoricSqlDoctorProject,
deps: HistoricSqlDoctorDeps = {},
): Promise<DoctorCheck[]> {
const targets = Object.entries(project.config.connections)
.filter(([, connection]) => isEnabledPostgresHistoricSql(connection))
.sort(([left], [right]) => left.localeCompare(right));
if (targets.length === 0) {
return [
check('pass', 'historic-sql-postgres', 'Postgres Historic SQL', 'No enabled Postgres historic-SQL connections'),
];
}
const probe = deps.postgresHistoricSqlProbe ?? defaultPostgresHistoricSqlProbe;
const env = deps.env ?? process.env;
const checks: DoctorCheck[] = [];
for (const [connectionId, connection] of targets) {
const label = `Postgres Historic SQL (${connectionId})`;
if (!isPostgresDriver(connection)) {
checks.push(
check(
'fail',
checkId(connectionId),
label,
`connections.${connectionId}.historicSql.dialect is postgres but driver is ${String(connection.driver)}`,
`Set connections.${connectionId}.driver to postgres or disable historicSql for this connection`,
),
);
continue;
}
try {
const result = await probe({ projectDir: project.projectDir, connectionId, connection, env });
if (result.warnings.length > 0) {
checks.push(
check(
'warn',
checkId(connectionId),
label,
`pg_stat_statements ready (${result.pgServerVersion}) with warnings: ${result.warnings.join('; ')}`,
`Update the Postgres parameter group or config, then rerun \`klo dev doctor --project-dir ${project.projectDir}\``,
),
);
} else {
checks.push(
check('pass', checkId(connectionId), label, `pg_stat_statements ready (${result.pgServerVersion})`),
);
}
} catch (error) {
checks.push(
check(
'fail',
checkId(connectionId),
label,
failureDetail(error),
capabilityFailureFix(error, connectionId, project.projectDir),
),
);
}
}
return checks;
}

File diff suppressed because it is too large Load diff

53
packages/cli/src/index.ts Normal file
View file

@ -0,0 +1,53 @@
import { profileMark } from './startup-profile.js';
export {
getKloCliPackageInfo,
runInitForCommander,
runKloCli,
type KloCliDeps,
type KloCliIo,
type KloCliPackageInfo,
} from './cli-runtime.js';
export { runKloAgent, type KloAgentArgs } from './agent.js';
export {
KLO_AGENT_MAX_ROWS_CAP,
createKloAgentRuntime,
parseAgentMaxRows,
readAgentJsonFile,
writeAgentJson,
writeAgentJsonError,
type KloAgentRuntime,
type KloAgentRuntimeDeps,
} from './agent-runtime.js';
export { runKloSetup, type KloSetupArgs, type KloSetupStatus } from './setup.js';
export type {
KloSetupDatabaseDriver,
KloSetupDatabasesArgs,
KloSetupDatabasesDeps,
KloSetupDatabasesResult,
} from './setup-databases.js';
export { runKloSetupDatabasesStep } from './setup-databases.js';
export type {
KloSetupEmbeddingBackend,
KloSetupEmbeddingsArgs,
KloSetupEmbeddingsDeps,
KloSetupEmbeddingsResult,
} from './setup-embeddings.js';
export { runKloSetupEmbeddingsStep } from './setup-embeddings.js';
export type {
KloSetupSourcesArgs,
KloSetupSourcesDeps,
KloSetupSourcesPromptAdapter,
KloSetupSourcesResult,
KloSetupSourceType,
} from './setup-sources.js';
export { runKloSetupSourcesStep } from './setup-sources.js';
export type { KloMemoryFlowTuiIo, MemoryFlowTuiLiveSession } from './memory-flow-tui.js';
export {
renderMemoryFlowTui,
sanitizeMemoryFlowTuiError,
startLiveMemoryFlowTui,
} from './memory-flow-tui.js';
export { rendererUnavailableVizFallback, resolveVizFallback, warnVizFallbackOnce } from './viz-fallback.js';
profileMark('module:index');

View file

@ -0,0 +1,78 @@
import { mkdtemp, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
import { readIngestReportSnapshotFile } from './ingest-report-file.js';
function reportSnapshot() {
return {
id: 'report-1',
runId: 'run-1',
jobId: 'job-1',
connectionId: 'warehouse',
sourceKey: 'metabase',
createdAt: '2026-04-30T12:00:00.000Z',
body: {
syncId: 'sync-1',
diffSummary: { added: 1, modified: 0, deleted: 0, unchanged: 0 },
commitSha: null,
workUnits: [],
failedWorkUnits: [],
reconciliationSkipped: true,
conflictsResolved: [],
evictionsApplied: [],
unmappedFallbacks: [],
evictionInputs: [],
unresolvedCards: [],
supersededBy: null,
overrideOf: null,
provenanceRows: [],
toolTranscripts: [],
},
};
}
describe('readIngestReportSnapshotFile', () => {
let tempDir: string;
beforeEach(async () => {
tempDir = await mkdtemp(join(tmpdir(), 'klo-report-file-'));
});
afterEach(async () => {
await rm(tempDir, { recursive: true, force: true });
});
it('reads and parses an ingest report JSON file', async () => {
const reportPath = join(tempDir, 'report.json');
await writeFile(reportPath, `${JSON.stringify(reportSnapshot(), null, 2)}\n`, 'utf-8');
const report = await readIngestReportSnapshotFile(reportPath);
expect(report).toMatchObject({
id: 'report-1',
runId: 'run-1',
jobId: 'job-1',
connectionId: 'warehouse',
sourceKey: 'metabase',
});
});
it('reports invalid JSON with the file path', async () => {
const reportPath = join(tempDir, 'invalid.json');
await writeFile(reportPath, '{not json', 'utf-8');
await expect(readIngestReportSnapshotFile(reportPath)).rejects.toThrow(
`Invalid JSON in ingest report file ${reportPath}`,
);
});
it('reports schema failures with the file path', async () => {
const reportPath = join(tempDir, 'wrong-shape.json');
await writeFile(reportPath, JSON.stringify({ id: 'report-1' }), 'utf-8');
await expect(readIngestReportSnapshotFile(reportPath)).rejects.toThrow(
`Invalid ingest report file ${reportPath}`,
);
});
});

View file

@ -0,0 +1,20 @@
import { readFile } from 'node:fs/promises';
import { parseIngestReportSnapshot, type IngestReportSnapshot } from '@klo/context/ingest';
export async function readIngestReportSnapshotFile(reportFile: string): Promise<IngestReportSnapshot> {
const raw = await readFile(reportFile, 'utf-8');
let parsed: unknown;
try {
parsed = JSON.parse(raw);
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
throw new Error(`Invalid JSON in ingest report file ${reportFile}: ${message}`);
}
try {
return parseIngestReportSnapshot(parsed);
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
throw new Error(`Invalid ingest report file ${reportFile}: ${message}`);
}
}

File diff suppressed because it is too large Load diff

425
packages/cli/src/ingest.ts Normal file
View file

@ -0,0 +1,425 @@
import {
buildMemoryFlowViewModel,
createMemoryFlowLiveBuffer,
formatMemoryFlowFinalSummary,
getLatestLocalIngestStatus,
getLocalIngestStatus,
type IngestReportSnapshot,
ingestReportToMemoryFlowReplay,
type LocalMetabaseFanoutResult,
type LocalMetabaseFanoutProgress,
type MemoryFlowReplayInput,
type RunLocalIngestOptions,
renderMemoryFlowReplay,
runLocalIngest,
runLocalMetabaseIngest,
} from '@klo/context/ingest';
import { loadKloProject } from '@klo/context/project';
import { readIngestReportSnapshotFile } from './ingest-report-file.js';
import { createKloCliLocalIngestAdapters } from './local-adapters.js';
import { type KloMemoryFlowStdin, renderMemoryFlowInteractively } from './memory-flow-interactive.js';
import {
type KloMemoryFlowTuiIo,
type MemoryFlowTuiLiveSession,
renderMemoryFlowTui,
startLiveMemoryFlowTui,
} from './memory-flow-tui.js';
import { resolveVizFallback, warnVizFallbackOnce } from './viz-fallback.js';
import { profileMark } from './startup-profile.js';
profileMark('module:ingest');
export type KloIngestOutputMode = 'plain' | 'json' | 'viz';
type KloIngestInputMode = 'auto' | 'disabled';
export type KloIngestArgs =
| {
command: 'run';
projectDir: string;
connectionId: string;
adapter: string;
sourceDir?: string;
databaseIntrospectionUrl?: string;
debugLlmRequestFile?: string;
outputMode: KloIngestOutputMode;
inputMode?: KloIngestInputMode;
}
| {
command: 'status' | 'replay' | 'watch';
projectDir: string;
runId?: string;
reportFile?: string;
outputMode: KloIngestOutputMode;
inputMode?: KloIngestInputMode;
};
interface KloIngestIo {
stdin?: KloMemoryFlowStdin;
stdout: { isTTY?: boolean; columns?: number; write(chunk: string): void };
stderr: { write(chunk: string): void };
}
interface KloIngestDeps {
jobIdFactory?: () => string;
now?: () => Date;
createAdapters?: typeof createKloCliLocalIngestAdapters;
runLocalIngest?: typeof runLocalIngest;
runLocalMetabaseIngest?: typeof runLocalMetabaseIngest;
readReportFile?: typeof readIngestReportSnapshotFile;
renderStoredMemoryFlow?: typeof renderMemoryFlowTui;
startLiveMemoryFlow?: typeof startLiveMemoryFlowTui;
env?: NodeJS.ProcessEnv;
localIngestOptions?: Pick<
RunLocalIngestOptions,
| 'agentRunner'
| 'llmProvider'
| 'memoryModel'
| 'semanticLayerCompute'
| 'queryExecutor'
| 'logger'
| 'pullConfigOptions'
>;
}
function reportStatus(report: IngestReportSnapshot): 'done' | 'error' {
return report.body.failedWorkUnits.length > 0 ? 'error' : 'done';
}
function reportActionCounts(report: IngestReportSnapshot): { wikiCount: number; slCount: number } {
const actions = report.body.workUnits.flatMap((workUnit) => workUnit.actions);
return {
wikiCount: actions.filter((action) => action.target === 'wiki').length,
slCount: actions.filter((action) => action.target === 'sl').length,
};
}
function writeReportStatus(report: IngestReportSnapshot, io: KloIngestIo): void {
const counts = reportActionCounts(report);
io.stdout.write(`Report: ${report.id}\n`);
io.stdout.write(`Run: ${report.runId}\n`);
io.stdout.write(`Job: ${report.jobId}\n`);
io.stdout.write(`Status: ${reportStatus(report)}\n`);
io.stdout.write(`Adapter: ${report.sourceKey}\n`);
io.stdout.write(`Connection: ${report.connectionId}\n`);
io.stdout.write(`Sync: ${report.body.syncId}\n`);
io.stdout.write(
`Diff: +${report.body.diffSummary.added}/~${report.body.diffSummary.modified}/-${report.body.diffSummary.deleted}/=${report.body.diffSummary.unchanged}\n`,
);
io.stdout.write(`Work units: ${report.body.workUnits.length}\n`);
io.stdout.write(`Saved memory: ${counts.wikiCount} wiki, ${counts.slCount} SL\n`);
io.stdout.write(`Provenance rows: ${report.body.provenanceRows.length}\n`);
}
function writeMetabaseFanoutStatus(result: LocalMetabaseFanoutResult, io: KloIngestIo): void {
io.stdout.write(`Metabase fan-out: ${result.status}\n`);
io.stdout.write(`Source: ${result.metabaseConnectionId}\n`);
io.stdout.write(`Children: ${result.children.length}\n`);
if (result.totals) {
io.stdout.write(`Work units: ${result.totals.workUnits}\n`);
io.stdout.write(`Failed work units: ${result.totals.failedWorkUnits}\n`);
}
for (const child of result.children) {
const status = reportStatus(child.report);
io.stdout.write(
`- target=${child.targetConnectionId} database=${child.metabaseDatabaseId} status=${status} job=${child.jobId}\n`,
);
}
}
function pluralize(count: number, singular: string, plural = `${singular}s`): string {
return `${count} ${count === 1 ? singular : plural}`;
}
function createMetabaseFanoutProgress(
connectionId: string,
io: KloIngestIo,
): LocalMetabaseFanoutProgress {
io.stdout.write(`Metabase ingest: ${connectionId}\n`);
io.stdout.write('Checking mappings and scheduled-pull targets...\n');
return {
onMetabaseFanoutPlanned(event) {
io.stdout.write(`Targets: ${pluralize(event.children.length, 'mapped database')}\n`);
for (const child of event.children) {
io.stdout.write(`- database=${child.metabaseDatabaseId} target=${child.targetConnectionId} status=queued\n`);
}
},
onMetabaseChildStarted(event) {
io.stdout.write(
`- database=${event.metabaseDatabaseId} target=${event.targetConnectionId} status=running job=${event.jobId}\n`,
);
},
onMetabaseChildCompleted(event) {
io.stdout.write(
`- database=${event.metabaseDatabaseId} target=${event.targetConnectionId} status=${event.status} job=${event.jobId}\n`,
);
},
};
}
function writeReportJson(report: IngestReportSnapshot, io: KloIngestIo): void {
io.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
}
function assertReportMatchesReplayId(report: IngestReportSnapshot, requestedId: string, reportFile: string): void {
const validIds = [report.id, report.runId, report.jobId];
if (!validIds.includes(requestedId)) {
throw new Error(
`Report file ${reportFile} does not match ingest replay id "${requestedId}"; expected one of ${validIds.join(
', ',
)}`,
);
}
}
async function readStoredIngestReport(
project: Awaited<ReturnType<typeof loadKloProject>>,
runId: string | undefined,
): Promise<IngestReportSnapshot | null> {
return runId ? await getLocalIngestStatus(project, runId) : await getLatestLocalIngestStatus(project);
}
function isInteractiveTerminal(io: KloIngestIo): boolean {
return io.stdout.isTTY === true;
}
function terminalWidth(io: KloIngestIo): number | undefined {
return io.stdout.columns ?? process.stdout.columns;
}
function isTuiCapableIo(io: KloIngestIo): io is KloIngestIo & KloMemoryFlowTuiIo {
return (
io.stdin?.isTTY === true &&
io.stdout.isTTY === true &&
typeof io.stdin.on === 'function' &&
typeof io.stdin.setRawMode === 'function' &&
typeof io.stdout.write === 'function'
);
}
interface EffectiveIngestOutputModeOptions {
requireInput?: boolean;
}
function effectiveIngestOutputMode(
outputMode: KloIngestOutputMode,
io: KloIngestIo,
env: NodeJS.ProcessEnv,
options: EffectiveIngestOutputModeOptions = {},
): KloIngestOutputMode {
if (outputMode !== 'viz') {
return outputMode;
}
const fallback = resolveVizFallback(io, env, { requireInput: options.requireInput ?? false });
if (!fallback.shouldDegrade) {
return outputMode;
}
warnVizFallbackOnce(io, fallback);
return 'plain';
}
function writeMemoryFlowInput(input: MemoryFlowReplayInput, io: KloIngestIo, options: { clear?: boolean } = {}): void {
if (options.clear) {
io.stdout.write('\u001b[2J\u001b[H');
}
const view = buildMemoryFlowViewModel(input);
io.stdout.write(renderMemoryFlowReplay(view, { terminalWidth: terminalWidth(io) }));
}
function initialRunMemoryFlowInput(
args: Extract<KloIngestArgs, { command: 'run' }>,
runId: string,
): MemoryFlowReplayInput {
return {
runId,
connectionId: args.connectionId,
adapter: args.adapter,
status: 'running',
sourceDir: args.sourceDir ?? null,
syncId: 'pending',
errors: [],
events: [],
plannedWorkUnits: [],
details: { actions: [], provenance: [], transcripts: [] },
};
}
async function writeReportRecord(
report: IngestReportSnapshot,
outputMode: KloIngestOutputMode,
io: KloIngestIo,
options: {
interactive?: boolean;
renderStoredMemoryFlow?: typeof renderMemoryFlowTui;
env?: NodeJS.ProcessEnv;
} = {},
): Promise<void> {
if (outputMode === 'json') {
writeReportJson(report, io);
return;
}
const resolvedOutputMode = effectiveIngestOutputMode(outputMode, io, options.env ?? process.env, {
requireInput: options.interactive === true,
});
if (resolvedOutputMode === 'viz') {
const input = ingestReportToMemoryFlowReplay(report, { provenanceRowCount: report.body.provenanceRows.length });
if (options.interactive === true) {
if (io.stdin?.isTTY === true) {
const renderStoredMemoryFlow = options.renderStoredMemoryFlow ?? renderMemoryFlowTui;
if (isTuiCapableIo(io) && (await renderStoredMemoryFlow(input, io))) {
return;
}
await renderMemoryFlowInteractively(input, io);
return;
}
writeMemoryFlowInput(input, io);
return;
}
writeMemoryFlowInput(input, io);
return;
}
writeReportStatus(report, io);
}
export async function runKloIngest(
args: KloIngestArgs,
io: KloIngestIo = process,
deps: KloIngestDeps = {},
): Promise<number> {
try {
const project = await loadKloProject({ projectDir: args.projectDir });
const env = deps.env ?? process.env;
if (args.command === 'run') {
const createAdapters = deps.createAdapters ?? createKloCliLocalIngestAdapters;
const executeLocalIngest = deps.runLocalIngest ?? runLocalIngest;
const localIngestOptions = deps.localIngestOptions ?? {};
const adapterOptions = {
...(localIngestOptions.pullConfigOptions ?? {}),
...(args.databaseIntrospectionUrl ? { databaseIntrospectionUrl: args.databaseIntrospectionUrl } : {}),
...(args.adapter === 'historic-sql' ? { historicSqlConnectionId: args.connectionId } : {}),
};
if (args.adapter === 'metabase' && args.sourceDir) {
throw new Error('source-dir uploads are not supported for the Metabase fan-out adapter');
}
if (args.adapter === 'metabase') {
const executeMetabaseFanout = deps.runLocalMetabaseIngest ?? runLocalMetabaseIngest;
const progress =
args.outputMode === 'json' ? undefined : createMetabaseFanoutProgress(args.connectionId, io);
const result = await executeMetabaseFanout({
project,
adapters: createAdapters(project, adapterOptions),
metabaseConnectionId: args.connectionId,
...localIngestOptions,
trigger: 'manual_resync',
jobIdFactory: deps.jobIdFactory,
...(progress ? { progress } : {}),
});
if (args.outputMode === 'json') {
io.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
} else {
writeMetabaseFanoutStatus(result, io);
}
return 0;
}
const jobId = deps.jobIdFactory?.();
let liveTui: MemoryFlowTuiLiveSession | null = null;
const runOutputMode = effectiveIngestOutputMode(args.outputMode, io, env, {
requireInput: (args.inputMode ?? 'auto') === 'auto',
});
const shouldUseLiveViz =
runOutputMode === 'viz' && (args.inputMode ?? 'auto') === 'auto' && isInteractiveTerminal(io);
const initialMemoryFlow = shouldUseLiveViz ? initialRunMemoryFlowInput(args, jobId ?? 'pending') : undefined;
let latestMemoryFlowSnapshot: MemoryFlowReplayInput | null = initialMemoryFlow ?? null;
if (initialMemoryFlow && isTuiCapableIo(io)) {
const startLiveMemoryFlow = deps.startLiveMemoryFlow ?? startLiveMemoryFlowTui;
liveTui = await startLiveMemoryFlow(initialMemoryFlow, io);
}
const memoryFlow = initialMemoryFlow
? createMemoryFlowLiveBuffer(initialMemoryFlow, {
onChange: (snapshot) => {
latestMemoryFlowSnapshot = snapshot;
if (liveTui && !liveTui.isClosed()) {
liveTui.update(snapshot);
return;
}
if (!liveTui) {
writeMemoryFlowInput(snapshot, io, { clear: true });
}
},
})
: undefined;
try {
const result = await executeLocalIngest({
project,
adapters: createAdapters(project, adapterOptions),
adapter: args.adapter,
connectionId: args.connectionId,
sourceDir: args.sourceDir,
trigger: 'manual_resync',
jobId,
...localIngestOptions,
...(args.debugLlmRequestFile ? { llmDebugRequestFile: args.debugLlmRequestFile } : {}),
...(memoryFlow ? { memoryFlow } : {}),
});
if (memoryFlow) {
latestMemoryFlowSnapshot = memoryFlow.snapshot();
liveTui?.close();
liveTui = null;
io.stdout.write(formatMemoryFlowFinalSummary(latestMemoryFlowSnapshot));
return 0;
}
await writeReportRecord(result.report, runOutputMode, io, {
interactive: (args.inputMode ?? 'auto') === 'auto',
renderStoredMemoryFlow: deps.renderStoredMemoryFlow,
env,
});
return 0;
} finally {
liveTui?.close();
}
}
if (args.reportFile) {
const readReportFile = deps.readReportFile ?? readIngestReportSnapshotFile;
const report = await readReportFile(args.reportFile);
if (args.runId) {
assertReportMatchesReplayId(report, args.runId, args.reportFile);
}
await writeReportRecord(report, args.outputMode, io, {
interactive: (args.inputMode ?? 'auto') === 'auto',
renderStoredMemoryFlow: deps.renderStoredMemoryFlow,
env,
});
return 0;
}
const report = await readStoredIngestReport(project, args.runId);
if (!report) {
throw new Error(
args.runId
? `Local ingest run or report "${args.runId}" was not found`
: 'No local ingest reports were found. Run `klo ingest --all` first.',
);
}
await writeReportRecord(report, args.outputMode, io, {
interactive: (args.inputMode ?? 'auto') === 'auto',
renderStoredMemoryFlow: deps.renderStoredMemoryFlow,
env,
});
return 0;
} catch (error) {
io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
return 1;
}
}

View file

@ -0,0 +1,60 @@
import { describe, expect, it } from 'vitest';
import type { KloCliIo } from '../cli-runtime.js';
import { resolveOutputMode } from './mode.js';
function ioWith(isTTY: boolean | undefined): KloCliIo {
return {
stdout: { isTTY, write: () => {} },
stderr: { write: () => {} },
};
}
describe('resolveOutputMode', () => {
it('uses explicit value when provided', () => {
expect(resolveOutputMode({ explicit: 'pretty', io: ioWith(false), env: {} })).toBe('pretty');
expect(resolveOutputMode({ explicit: 'plain', io: ioWith(true), env: {} })).toBe('plain');
expect(resolveOutputMode({ explicit: 'json', io: ioWith(true), env: {} })).toBe('json');
});
it('json:true takes precedence over explicit value', () => {
expect(resolveOutputMode({ explicit: 'pretty', json: true, io: ioWith(true), env: {} })).toBe('json');
});
it('throws on unknown explicit value', () => {
expect(() => resolveOutputMode({ explicit: 'fancy', io: ioWith(true), env: {} })).toThrow(/Invalid --output/);
});
it('honors KLO_OUTPUT env var when no explicit value', () => {
expect(resolveOutputMode({ io: ioWith(true), env: { KLO_OUTPUT: 'plain' } })).toBe('plain');
expect(resolveOutputMode({ io: ioWith(false), env: { KLO_OUTPUT: 'pretty' } })).toBe('pretty');
expect(resolveOutputMode({ io: ioWith(false), env: { KLO_OUTPUT: 'json' } })).toBe('json');
});
it('throws on unknown KLO_OUTPUT', () => {
expect(() => resolveOutputMode({ io: ioWith(true), env: { KLO_OUTPUT: 'fancy' } })).toThrow(/Invalid KLO_OUTPUT/);
});
it('returns plain when CI is set to a truthy value', () => {
expect(resolveOutputMode({ io: ioWith(true), env: { CI: 'true' } })).toBe('plain');
expect(resolveOutputMode({ io: ioWith(true), env: { CI: '1' } })).toBe('plain');
});
it('ignores CI when set to a falsy value', () => {
expect(resolveOutputMode({ io: ioWith(true), env: { CI: '' } })).toBe('pretty');
expect(resolveOutputMode({ io: ioWith(true), env: { CI: '0' } })).toBe('pretty');
expect(resolveOutputMode({ io: ioWith(true), env: { CI: 'false' } })).toBe('pretty');
});
it('returns pretty when stdout is a TTY and CI is not set', () => {
expect(resolveOutputMode({ io: ioWith(true), env: {} })).toBe('pretty');
});
it('returns plain when stdout is not a TTY', () => {
expect(resolveOutputMode({ io: ioWith(false), env: {} })).toBe('plain');
expect(resolveOutputMode({ io: ioWith(undefined), env: {} })).toBe('plain');
});
it('explicit value beats KLO_OUTPUT env var', () => {
expect(resolveOutputMode({ explicit: 'json', io: ioWith(true), env: { KLO_OUTPUT: 'plain' } })).toBe('json');
});
});

View file

@ -0,0 +1,40 @@
import type { KloCliIo } from '../cli-runtime.js';
export type KloOutputMode = 'pretty' | 'plain' | 'json';
const MODES: ReadonlySet<string> = new Set(['pretty', 'plain', 'json']);
export interface ResolveOutputModeArgs {
explicit?: string;
json?: boolean;
io: KloCliIo;
env?: NodeJS.ProcessEnv;
}
export function resolveOutputMode(args: ResolveOutputModeArgs): KloOutputMode {
if (args.json === true) {
return 'json';
}
if (args.explicit !== undefined) {
if (!MODES.has(args.explicit)) {
throw new Error(`Invalid --output value: ${args.explicit}. Expected one of pretty, plain, json.`);
}
return args.explicit as KloOutputMode;
}
const env = args.env ?? process.env;
const envMode = env.KLO_OUTPUT;
if (envMode !== undefined && envMode !== '') {
if (!MODES.has(envMode)) {
throw new Error(`Invalid KLO_OUTPUT value: ${envMode}. Expected one of pretty, plain, json.`);
}
return envMode as KloOutputMode;
}
const ci = env.CI;
if (ci !== undefined && ci !== '' && ci !== '0' && ci !== 'false') {
return 'plain';
}
if (args.io.stdout.isTTY === true) {
return 'pretty';
}
return 'plain';
}

View file

@ -0,0 +1,171 @@
import { describe, expect, it } from 'vitest';
import type { KloCliIo } from '../cli-runtime.js';
import { printList, type PrintListColumn } from './print-list.js';
import { SYMBOLS } from './symbols.js';
function recorder(): { io: KloCliIo; out: () => string; err: () => string } {
let stdout = '';
let stderr = '';
return {
io: {
stdout: { write: (chunk: string) => { stdout += chunk; } },
stderr: { write: (chunk: string) => { stderr += chunk; } },
},
out: () => stdout,
err: () => stderr,
};
}
interface SlRow {
connectionId: string;
name: string;
columnCount: number;
measureCount: number;
joinCount: number;
description?: string;
}
const SL_COLUMNS: ReadonlyArray<PrintListColumn<SlRow>> = [
{ key: 'connectionId', label: 'CONNECTION', plain: '' },
{ key: 'name', label: 'NAME', plain: '' },
{ key: 'columnCount', label: 'COLS', plain: 'columns=', dim: true },
{ key: 'measureCount', label: 'MEASURES', plain: 'measures=', dim: true },
{ key: 'joinCount', label: 'JOINS', plain: 'joins=', dim: true },
{ key: 'description', label: 'DESCRIPTION', plain: false, optional: true, dim: true },
];
const ORDERS: SlRow = { connectionId: 'warehouse', name: 'orders', columnCount: 5, measureCount: 3, joinCount: 1 };
const USERS: SlRow = { connectionId: 'warehouse', name: 'users', columnCount: 8, measureCount: 2, joinCount: 2, description: 'User profile + auth' };
describe('printList — plain mode', () => {
it('emits one tab-separated row per item, skipping plain:false columns', () => {
const r = recorder();
printList<SlRow>({
rows: [ORDERS, USERS],
columns: SL_COLUMNS,
mode: 'plain',
command: 'sl list',
emptyMessage: 'No sources',
io: r.io,
});
expect(r.out()).toBe(
'warehouse\torders\tcolumns=5\tmeasures=3\tjoins=1\n' +
'warehouse\tusers\tcolumns=8\tmeasures=2\tjoins=2\n',
);
});
it('emits nothing on empty list (preserves current sl list zero-row behavior)', () => {
const r = recorder();
printList<SlRow>({
rows: [],
columns: SL_COLUMNS,
mode: 'plain',
command: 'sl list',
emptyMessage: 'No sources',
io: r.io,
});
expect(r.out()).toBe('');
});
});
describe('printList — json mode', () => {
it('emits the envelope with kind=list, data.items, and meta.command', () => {
const r = recorder();
printList<SlRow>({
rows: [ORDERS, USERS],
columns: SL_COLUMNS,
mode: 'json',
command: 'sl list',
emptyMessage: 'No sources',
io: r.io,
});
const written = r.out();
expect(written.endsWith('\n')).toBe(true);
const parsed = JSON.parse(written);
expect(parsed).toEqual({
kind: 'list',
data: { items: [ORDERS, USERS] },
meta: { command: 'sl list' },
});
});
it('emits an empty items array when no rows', () => {
const r = recorder();
printList<SlRow>({
rows: [],
columns: SL_COLUMNS,
mode: 'json',
command: 'sl list',
emptyMessage: 'No sources',
io: r.io,
});
expect(JSON.parse(r.out())).toEqual({
kind: 'list',
data: { items: [] },
meta: { command: 'sl list' },
});
});
});
function stripAnsi(s: string): string {
// Matches ESC [ ... m sequences emitted by node:util.styleText.
return s.replace(/\[[0-9;]*m/g, '');
}
describe('printList — pretty mode', () => {
it('renders a Clack-style header, grouped rows, and footer', () => {
const r = recorder();
printList<SlRow>({
rows: [ORDERS, USERS],
columns: SL_COLUMNS,
groupBy: 'connectionId',
mode: 'pretty',
command: 'sl list',
emptyMessage: 'No sources',
io: r.io,
});
const out = stripAnsi(r.out());
expect(out).toContain(`${SYMBOLS.barStart} sl list`);
expect(out).toContain(`${SYMBOLS.group} warehouse`);
expect(out).toContain('(2 sources)');
expect(out).toMatch(new RegExp(`${escapeRegExp(SYMBOLS.item)} orders\\s+5 cols ${escapeRegExp(SYMBOLS.middot)} 3 measures ${escapeRegExp(SYMBOLS.middot)} 1 join\\b`));
expect(out).toMatch(new RegExp(`${escapeRegExp(SYMBOLS.item)} users\\s+8 cols ${escapeRegExp(SYMBOLS.middot)} 2 measures ${escapeRegExp(SYMBOLS.middot)} 2 joins\\b`));
expect(out).toContain(`${SYMBOLS.emDash} User profile + auth`);
expect(out).toContain(`${SYMBOLS.barEnd} 2 sources`);
});
it('renders an empty-state message when no rows', () => {
const r = recorder();
printList<SlRow>({
rows: [],
columns: SL_COLUMNS,
groupBy: 'connectionId',
mode: 'pretty',
command: 'sl list',
emptyMessage: 'No semantic-layer sources found in /tmp/proj',
io: r.io,
});
const out = stripAnsi(r.out());
expect(out).toContain(`${SYMBOLS.barStart} sl list`);
expect(out).toContain(`${SYMBOLS.barEnd} No semantic-layer sources found in /tmp/proj`);
});
it('singularizes the footer when there is one row', () => {
const r = recorder();
printList<SlRow>({
rows: [ORDERS],
columns: SL_COLUMNS,
groupBy: 'connectionId',
mode: 'pretty',
command: 'sl list',
emptyMessage: 'No sources',
io: r.io,
});
const out = stripAnsi(r.out());
expect(out).toContain(`${SYMBOLS.barEnd} 1 source`);
});
});
function escapeRegExp(s: string): string {
return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
}

View file

@ -0,0 +1,164 @@
import type { KloCliIo } from '../cli-runtime.js';
import type { KloOutputMode } from './mode.js';
import { bold, dim, SYMBOLS } from './symbols.js';
export interface PrintListColumn<Row> {
key: keyof Row & string;
label?: string;
/**
* Plain-mode rendering control.
* - `string` (including `''`): emit `${plain}${value}` as a tab-separated cell.
* - `false`: omit this column entirely in plain mode.
* - `undefined`: same as `''`.
*/
plain?: string | false;
/** Skip this column when the row's value is null / undefined / empty string. */
optional?: boolean;
/** Pretty-mode hint: render this column dim. */
dim?: boolean;
}
export interface PrintListArgs<Row> {
rows: ReadonlyArray<Row>;
columns: ReadonlyArray<PrintListColumn<Row>>;
groupBy?: keyof Row & string;
emptyMessage: string;
command: string;
mode: KloOutputMode;
io: KloCliIo;
}
export function printList<Row extends object>(args: PrintListArgs<Row>): void {
switch (args.mode) {
case 'json':
printListJson(args);
return;
case 'plain':
printListPlain(args);
return;
case 'pretty':
printListPretty(args);
return;
}
}
function isEmpty(value: unknown): boolean {
return value === undefined || value === null || value === '';
}
function printListPlain<Row extends object>(args: PrintListArgs<Row>): void {
for (const row of args.rows) {
const cells: string[] = [];
for (const col of args.columns) {
if (col.plain === false) continue;
const value = row[col.key];
if (col.optional && isEmpty(value)) continue;
const prefix = col.plain ?? '';
cells.push(`${prefix}${value === undefined || value === null ? '' : String(value)}`);
}
args.io.stdout.write(`${cells.join('\t')}\n`);
}
}
function printListJson<Row extends object>(args: PrintListArgs<Row>): void {
const envelope = {
kind: 'list',
data: { items: args.rows },
meta: { command: args.command },
};
args.io.stdout.write(`${JSON.stringify(envelope, null, 2)}\n`);
}
function pluralize(count: number, singular: string): string {
return `${count} ${count === 1 ? singular : `${singular}s`}`;
}
function metricCell(label: string, count: number): string {
// "5 cols", "3 measures", "1 join" / "2 joins"
// The label in PrintListColumn is uppercase; pretty mode lowercases it.
const word = label.toLowerCase();
return `${count} ${count === 1 ? singularize(word) : word}`;
}
function singularize(word: string): string {
if (word === 'joins') return 'join';
if (word === 'measures') return 'measure';
if (word === 'cols') return 'col';
if (word.endsWith('s')) return word.slice(0, -1);
return word;
}
function groupRows<Row extends object>(
rows: ReadonlyArray<Row>,
key: keyof Row & string,
): Map<string, Row[]> {
const groups = new Map<string, Row[]>();
for (const row of rows) {
const value = String(row[key] ?? '');
const bucket = groups.get(value);
if (bucket) {
bucket.push(row);
} else {
groups.set(value, [row]);
}
}
return groups;
}
function printListPretty<Row extends object>(args: PrintListArgs<Row>): void {
const { io, command, rows, columns, groupBy, emptyMessage } = args;
io.stdout.write(`${SYMBOLS.barStart} ${command}\n`);
io.stdout.write(`${SYMBOLS.bar}\n`);
if (rows.length === 0) {
io.stdout.write(`${SYMBOLS.barEnd} ${emptyMessage}\n`);
return;
}
// Identify role of each column.
// - First non-grouped, non-metric, non-optional column = "name" column (bolded)
// - Columns with a `plain` prefix = metric columns (rendered as "N word")
// - optional columns = trailing suffix (em-dash + value), only when value is present
const nameCol = columns.find(
(c) => c.key !== groupBy && !c.plain && !c.optional && c.plain !== false,
);
const metricCols = columns.filter((c) => typeof c.plain === 'string' && c.plain.length > 0);
const optionalCols = columns.filter((c) => c.optional === true);
const buckets = groupBy ? groupRows(rows, groupBy) : new Map<string, Row[]>([['', [...rows]]]);
const nameWidth = nameCol
? Math.max(...rows.map((r) => String(r[nameCol.key] ?? '').length))
: 0;
for (const [groupValue, groupRowList] of buckets) {
if (groupBy) {
io.stdout.write(
`${SYMBOLS.bar} ${SYMBOLS.group} ${bold(groupValue)} ${dim(`(${pluralize(groupRowList.length, 'source')})`)}\n`,
);
}
for (const row of groupRowList) {
const segments: string[] = [];
if (nameCol) {
segments.push(String(row[nameCol.key] ?? '').padEnd(nameWidth));
}
const metrics = metricCols
.map((c) => metricCell(c.label ?? c.key, Number(row[c.key] ?? 0)))
.join(` ${SYMBOLS.middot} `);
if (metrics.length > 0) segments.push(dim(metrics));
const optionalSuffix = optionalCols
.map((c) => row[c.key])
.filter((v) => !isEmpty(v))
.map((v) => `${SYMBOLS.emDash} ${dim(String(v))}`)
.join(' ');
if (optionalSuffix.length > 0) segments.push(optionalSuffix);
const indent = groupBy ? ' ' : ' ';
io.stdout.write(`${SYMBOLS.bar}${indent}${SYMBOLS.item} ${segments.join(' ')}\n`);
}
}
io.stdout.write(`${SYMBOLS.bar}\n`);
io.stdout.write(`${SYMBOLS.barEnd} ${pluralize(rows.length, 'source')}\n`);
}

View file

@ -0,0 +1,37 @@
import { styleText } from 'node:util';
function detectUnicodeSupport(env: NodeJS.ProcessEnv = process.env): boolean {
if (process.platform !== 'win32') {
return env.TERM !== 'linux';
}
return (
Boolean(env.WT_SESSION) ||
env.TERM_PROGRAM === 'vscode' ||
env.TERM === 'xterm-256color' ||
env.TERM === 'alacritty'
);
}
const unicode = detectUnicodeSupport();
export const SYMBOLS = {
bar: unicode ? '│' : '|',
barStart: unicode ? '◇' : 'o',
barEnd: unicode ? '└' : '—',
group: unicode ? '●' : '*',
item: unicode ? '◆' : '*',
middot: unicode ? '·' : '-',
emDash: unicode ? '—' : '--',
} as const;
export function dim(text: string): string {
return styleText('dim', text);
}
export function bold(text: string): string {
return styleText('bold', text);
}
export function gray(text: string): string {
return styleText('gray', text);
}

View file

@ -0,0 +1,95 @@
import { mkdtemp, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { initKloProject } from '@klo/context/project';
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
import { runKloKnowledge } from './knowledge.js';
function makeIo() {
let stdout = '';
let stderr = '';
return {
io: {
stdout: {
write: (chunk: string) => {
stdout += chunk;
},
},
stderr: {
write: (chunk: string) => {
stderr += chunk;
},
},
},
stdout: () => stdout,
stderr: () => stderr,
};
}
describe('runKloKnowledge', () => {
let tempDir: string;
beforeEach(async () => {
tempDir = await mkdtemp(join(tmpdir(), 'klo-cli-knowledge-'));
});
afterEach(async () => {
await rm(tempDir, { recursive: true, force: true });
});
it('writes, reads, lists, and searches knowledge pages', async () => {
const projectDir = join(tempDir, 'project');
await initKloProject({ projectDir, projectName: 'warehouse' });
const writeIo = makeIo();
await expect(
runKloKnowledge(
{
command: 'write',
projectDir,
key: 'metrics/revenue',
scope: 'GLOBAL',
userId: 'local',
summary: 'Revenue',
content: 'Revenue is paid order value.',
tags: ['finance'],
refs: [],
slRefs: ['orders'],
},
writeIo.io,
),
).resolves.toBe(0);
expect(writeIo.stdout()).toContain('Wrote knowledge/global/metrics/revenue.md');
const readIo = makeIo();
await expect(
runKloKnowledge({ command: 'read', projectDir, key: 'metrics/revenue', userId: 'local' }, readIo.io),
).resolves.toBe(0);
expect(readIo.stdout()).toContain('# metrics/revenue');
expect(readIo.stdout()).toContain('Revenue is paid order value.');
const listIo = makeIo();
await expect(runKloKnowledge({ command: 'list', projectDir, userId: 'local' }, listIo.io)).resolves.toBe(0);
expect(listIo.stdout()).toContain('GLOBAL\tmetrics/revenue\tRevenue');
const searchIo = makeIo();
await expect(
runKloKnowledge({ command: 'search', projectDir, query: 'paid order', userId: 'local' }, searchIo.io),
).resolves.toBe(0);
expect(searchIo.stdout()).toContain('metrics/revenue');
});
it('explains empty search results for a project without wiki pages', async () => {
const projectDir = join(tempDir, 'empty-project');
await initKloProject({ projectDir, projectName: 'warehouse' });
const searchIo = makeIo();
await expect(
runKloKnowledge({ command: 'search', projectDir, query: 'revenue', userId: 'local' }, searchIo.io),
).resolves.toBe(0);
expect(searchIo.stdout()).toBe('');
expect(searchIo.stderr()).toContain('No local wiki pages found');
expect(searchIo.stderr()).toContain('klo wiki write');
});
});

View file

@ -0,0 +1,90 @@
import { loadKloProject } from '@klo/context/project';
import {
type LocalKnowledgeScope,
listLocalKnowledgePages,
readLocalKnowledgePage,
searchLocalKnowledgePages,
writeLocalKnowledgePage,
} from '@klo/context/wiki';
export type KloKnowledgeArgs =
| { command: 'list'; projectDir: string; userId: string }
| { command: 'read'; projectDir: string; key: string; userId: string }
| { command: 'search'; projectDir: string; query: string; userId: string }
| {
command: 'write';
projectDir: string;
key: string;
scope: LocalKnowledgeScope;
userId: string;
summary: string;
content: string;
tags: string[];
refs: string[];
slRefs: string[];
};
interface KloKnowledgeIo {
stdout: { write(chunk: string): void };
stderr: { write(chunk: string): void };
}
export async function runKloKnowledge(args: KloKnowledgeArgs, io: KloKnowledgeIo = process): Promise<number> {
try {
const project = await loadKloProject({ projectDir: args.projectDir });
if (args.command === 'list') {
const pages = await listLocalKnowledgePages(project, { userId: args.userId });
for (const page of pages) {
io.stdout.write(`${page.scope}\t${page.key}\t${page.summary}\n`);
}
return 0;
}
if (args.command === 'read') {
const page = await readLocalKnowledgePage(project, { key: args.key, userId: args.userId });
if (!page) {
throw new Error(`Knowledge page "${args.key}" was not found`);
}
io.stdout.write(`# ${page.key}\n\n`);
io.stdout.write(`Scope: ${page.scope}\n`);
io.stdout.write(`Summary: ${page.summary}\n\n`);
io.stdout.write(`${page.content}\n`);
return 0;
}
if (args.command === 'search') {
const results = await searchLocalKnowledgePages(project, { query: args.query, userId: args.userId });
if (results.length === 0) {
const pages = await listLocalKnowledgePages(project, { userId: args.userId });
if (pages.length === 0) {
io.stderr.write(
`No local wiki pages found in ${project.projectDir}. Create one with \`klo wiki write <key> --summary <summary> --content <content>\` or run ingest.\n`,
);
} else {
io.stderr.write(
`No local wiki pages matched "${args.query}". Run \`klo wiki list\` to inspect available pages.\n`,
);
}
return 0;
}
for (const result of results) {
io.stdout.write(`${result.score}\t${result.scope}\t${result.key}\t${result.summary}\n`);
}
return 0;
}
const write = await writeLocalKnowledgePage(project, {
key: args.key,
scope: args.scope,
userId: args.userId,
summary: args.summary,
content: args.content,
tags: args.tags,
refs: args.refs,
slRefs: args.slRefs,
});
io.stdout.write(`Wrote ${write.path}\n`);
return 0;
} catch (error) {
io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
return 1;
}
}

View file

@ -0,0 +1,173 @@
import { join } from 'node:path';
import { createBigQueryLiveDatabaseIntrospection, isKloBigQueryConnectionConfig } from '@klo/connector-bigquery';
import { createClickHouseLiveDatabaseIntrospection, isKloClickHouseConnectionConfig } from '@klo/connector-clickhouse';
import { createMysqlLiveDatabaseIntrospection, isKloMysqlConnectionConfig } from '@klo/connector-mysql';
import {
createPostgresLiveDatabaseIntrospection,
isKloPostgresConnectionConfig,
type KloPostgresConnectionConfig,
KloPostgresHistoricSqlQueryClient,
} from '@klo/connector-postgres';
import { createSqliteLiveDatabaseIntrospection, isKloSqliteConnectionConfig } from '@klo/connector-sqlite';
import { createSqlServerLiveDatabaseIntrospection, isKloSqlServerConnectionConfig } from '@klo/connector-sqlserver';
import {
createDaemonLiveDatabaseIntrospection,
createDefaultLocalIngestAdapters,
type DefaultLocalIngestAdaptersOptions,
type LiveDatabaseIntrospectionPort,
LiveDatabaseSourceAdapter,
type SourceAdapter,
} from '@klo/context/ingest';
import type { KloLocalProject } from '@klo/context/project';
import { createHttpSqlAnalysisPort } from '@klo/context/sql-analysis';
function hasSnowflakeDriver(connection: unknown): boolean {
return (
typeof connection === 'object' &&
connection !== null &&
String((connection as { driver?: unknown }).driver ?? '').toLowerCase() === 'snowflake'
);
}
function createKloCliLiveDatabaseIntrospection(
project: KloLocalProject,
options: DefaultLocalIngestAdaptersOptions = {},
): LiveDatabaseIntrospectionPort {
const daemon = createDaemonLiveDatabaseIntrospection({
connections: project.config.connections,
...options.databaseIntrospection,
...(options.databaseIntrospectionUrl ? { baseUrl: options.databaseIntrospectionUrl } : {}),
});
const sqlite = createSqliteLiveDatabaseIntrospection({
projectDir: project.projectDir,
connections: project.config.connections,
});
const mysql = createMysqlLiveDatabaseIntrospection({
connections: project.config.connections,
});
const postgres = createPostgresLiveDatabaseIntrospection({
connections: project.config.connections,
});
const clickhouse = createClickHouseLiveDatabaseIntrospection({
connections: project.config.connections,
});
const sqlserver = createSqlServerLiveDatabaseIntrospection({
connections: project.config.connections,
});
const bigquery = createBigQueryLiveDatabaseIntrospection({
connections: project.config.connections,
});
return {
async extractSchema(connectionId: string) {
const connection = project.config.connections[connectionId];
if (isKloPostgresConnectionConfig(connection)) {
return postgres.extractSchema(connectionId);
}
if (isKloSqliteConnectionConfig(connection)) {
return sqlite.extractSchema(connectionId);
}
if (isKloMysqlConnectionConfig(connection)) {
return mysql.extractSchema(connectionId);
}
if (isKloClickHouseConnectionConfig(connection)) {
return clickhouse.extractSchema(connectionId);
}
if (isKloSqlServerConnectionConfig(connection)) {
return sqlserver.extractSchema(connectionId);
}
if (isKloBigQueryConnectionConfig(connection)) {
return bigquery.extractSchema(connectionId);
}
if (hasSnowflakeDriver(connection)) {
const { createSnowflakeLiveDatabaseIntrospection, isKloSnowflakeConnectionConfig } = await import(
'@klo/connector-snowflake'
);
if (!isKloSnowflakeConnectionConfig(connection)) {
return daemon.extractSchema(connectionId);
}
const snowflake = createSnowflakeLiveDatabaseIntrospection({
connections: project.config.connections,
});
return snowflake.extractSchema(connectionId);
}
return daemon.extractSchema(connectionId);
},
};
}
interface KloCliLocalIngestAdaptersOptions extends DefaultLocalIngestAdaptersOptions {
historicSqlConnectionId?: string;
sqlAnalysisUrl?: string;
}
function isEnabledPostgresHistoricSqlConnection(connection: KloPostgresConnectionConfig | undefined): boolean {
if (!connection || !isKloPostgresConnectionConfig(connection)) {
return false;
}
const historicSql =
typeof connection.historicSql === 'object' &&
connection.historicSql !== null &&
!Array.isArray(connection.historicSql)
? (connection.historicSql as Record<string, unknown>)
: null;
return historicSql?.enabled === true && historicSql.dialect === 'postgres';
}
function createEphemeralPostgresHistoricSqlClient(project: KloLocalProject, connectionId: string) {
const connection = project.config.connections[connectionId] as KloPostgresConnectionConfig | undefined;
if (!isKloPostgresConnectionConfig(connection)) {
throw new Error(
`Historic SQL local ingest requires a Postgres connection, got ${String(connection?.driver ?? 'unknown')}`,
);
}
return {
async executeQuery(sql: string, params?: unknown[]) {
const client = new KloPostgresHistoricSqlQueryClient({
connectionId,
connection,
});
try {
return await client.executeQuery(sql, params);
} finally {
await client.cleanup();
}
},
};
}
function historicSqlOptionsForLocalRun(project: KloLocalProject, options: KloCliLocalIngestAdaptersOptions) {
const connectionId = options.historicSqlConnectionId;
if (!connectionId) {
return undefined;
}
const connection = project.config.connections[connectionId] as KloPostgresConnectionConfig | undefined;
if (!isEnabledPostgresHistoricSqlConnection(connection)) {
return undefined;
}
return {
sqlAnalysis: createHttpSqlAnalysisPort({
baseUrl:
options.sqlAnalysisUrl ??
process.env.KLO_SQL_ANALYSIS_URL ??
process.env.KLO_DAEMON_URL ??
'http://127.0.0.1:8765',
}),
postgresQueryClient: createEphemeralPostgresHistoricSqlClient(project, connectionId),
postgresBaselineRootDir: join(project.projectDir, '.klo/cache/historic-sql'),
};
}
export function createKloCliLocalIngestAdapters(
project: KloLocalProject,
options: KloCliLocalIngestAdaptersOptions = {},
): SourceAdapter[] {
const historicSql = historicSqlOptionsForLocalRun(project, options);
const base = createDefaultLocalIngestAdapters(project, {
...options,
...(historicSql ? { historicSql } : {}),
});
const liveDatabase = new LiveDatabaseSourceAdapter({
introspection: createKloCliLiveDatabaseIntrospection(project, options),
});
return base.map((adapter) => (adapter.source === 'live-database' ? liveDatabase : adapter));
}

View file

@ -0,0 +1,163 @@
import { mkdtemp, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { initKloProject, loadKloProject } from '@klo/context/project';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { createKloCliScanConnector } from './local-scan-connectors.js';
const bigQueryMock = vi.hoisted(() => ({
constructorInputs: [] as Array<{
connectionId: string;
connection: unknown;
maxBytesBilled?: number | string;
}>,
}));
vi.mock('@klo/connector-bigquery', () => ({
isKloBigQueryConnectionConfig: (connection: { driver?: unknown } | undefined) =>
String(connection?.driver ?? '').toLowerCase() === 'bigquery',
KloBigQueryScanConnector: class {
readonly id: string;
readonly driver = 'bigquery';
constructor(options: { connectionId: string; connection: unknown; maxBytesBilled?: number | string }) {
bigQueryMock.constructorInputs.push(options);
this.id = `bigquery:${options.connectionId}`;
}
},
}));
describe('createKloCliScanConnector', () => {
let tempDir: string;
beforeEach(async () => {
bigQueryMock.constructorInputs.length = 0;
tempDir = await mkdtemp(join(tmpdir(), 'klo-cli-scan-connector-'));
});
afterEach(async () => {
await rm(tempDir, { recursive: true, force: true });
});
it('creates a native sqlite connector from standalone config', async () => {
await initKloProject({ projectDir: tempDir, projectName: 'warehouse' });
await writeFile(
join(tempDir, 'klo.yaml'),
[
'project: warehouse',
'connections:',
' warehouse:',
' driver: sqlite',
' path: warehouse.db',
' readonly: true',
'',
].join('\n'),
'utf-8',
);
const project = await loadKloProject({ projectDir: tempDir });
const connector = await createKloCliScanConnector(project, 'warehouse');
expect(connector.id).toBe('sqlite:warehouse');
expect(connector.driver).toBe('sqlite');
});
it.each([
['maxBytesBilled', ' maxBytesBilled: 123456789', 123456789],
['max_bytes_billed', ' max_bytes_billed: "987654321"', '987654321'],
])('passes BigQuery %s from standalone config', async (_label, byteCapLine, expectedMaxBytesBilled) => {
await initKloProject({ projectDir: tempDir, projectName: 'warehouse' });
await writeFile(
join(tempDir, 'klo.yaml'),
[
'project: warehouse',
'connections:',
' warehouse:',
' driver: bigquery',
' dataset_id: analytics',
' readonly: true',
byteCapLine,
'',
].join('\n'),
'utf-8',
);
const project = await loadKloProject({ projectDir: tempDir });
const connector = await createKloCliScanConnector(project, 'warehouse');
expect(connector.id).toBe('bigquery:warehouse');
expect(connector.driver).toBe('bigquery');
expect(bigQueryMock.constructorInputs).toEqual([
expect.objectContaining({
connectionId: 'warehouse',
maxBytesBilled: expectedMaxBytesBilled,
}),
]);
});
it('does not create a standalone PostHog scan connector', async () => {
await initKloProject({ projectDir: tempDir, projectName: 'warehouse' });
await writeFile(
join(tempDir, 'klo.yaml'),
[
'project: warehouse',
'connections:',
' product:',
' driver: posthog',
' api_key: phx_test',
' project_id: "157881"',
' readonly: true',
'',
].join('\n'),
'utf-8',
);
const project = await loadKloProject({ projectDir: tempDir });
await expect(createKloCliScanConnector(project, 'product')).rejects.toThrow(
'Connection "product" uses driver "posthog", which has no native standalone KLO scan connector',
);
});
it('throws for structural daemon-only fallback configs', async () => {
await initKloProject({ projectDir: tempDir, projectName: 'warehouse' });
await writeFile(
join(tempDir, 'klo.yaml'),
[
'project: warehouse',
'connections:',
' warehouse:',
' driver: duckdb',
' path: warehouse.duckdb',
'',
].join('\n'),
'utf-8',
);
const project = await loadKloProject({ projectDir: tempDir });
await expect(createKloCliScanConnector(project, 'warehouse')).rejects.toThrow(
'Connection "warehouse" uses driver "duckdb", which has no native standalone KLO scan connector',
);
});
it('throws a clear error when the connection block has no driver field', async () => {
await initKloProject({ projectDir: tempDir, projectName: 'warehouse' });
await writeFile(
join(tempDir, 'klo.yaml'),
[
'project: warehouse',
'connections:',
' warehouse:',
' type: postgres',
' url: postgresql://example/db',
' readonly: true',
'',
].join('\n'),
'utf-8',
);
const project = await loadKloProject({ projectDir: tempDir });
await expect(createKloCliScanConnector(project, 'warehouse')).rejects.toThrow(
'Connection "warehouse" has no `driver` field in klo.yaml',
);
});
});

View file

@ -0,0 +1,84 @@
import type { KloLocalProject } from '@klo/context/project';
import type { KloScanConnector } from '@klo/context/scan';
const SUPPORTED_DRIVERS = 'sqlite, postgres, mysql, clickhouse, sqlserver, bigquery, snowflake';
function bigQueryMaxBytesBilled(
connection: KloLocalProject['config']['connections'][string],
): number | string | undefined {
const raw = connection.maxBytesBilled ?? connection.max_bytes_billed;
if (typeof raw === 'number') {
return Number.isFinite(raw) && raw > 0 ? raw : undefined;
}
if (typeof raw === 'string') {
const trimmed = raw.trim();
return trimmed.length > 0 ? trimmed : undefined;
}
return undefined;
}
export async function createKloCliScanConnector(
project: KloLocalProject,
connectionId: string,
): Promise<KloScanConnector> {
const connection = project.config.connections[connectionId];
if (!connection) {
throw new Error(`Connection "${connectionId}" is not configured in klo.yaml`);
}
const driver = String(connection.driver ?? '').toLowerCase();
if (!driver) {
throw new Error(
`Connection "${connectionId}" has no \`driver\` field in klo.yaml. Supported drivers: ${SUPPORTED_DRIVERS}.`,
);
}
if (driver === 'sqlite' || driver === 'sqlite3') {
const { KloSqliteScanConnector, isKloSqliteConnectionConfig } = await import('@klo/connector-sqlite');
if (isKloSqliteConnectionConfig(connection)) {
return new KloSqliteScanConnector({ connectionId, connection, projectDir: project.projectDir });
}
}
if (driver === 'postgres' || driver === 'postgresql') {
const { KloPostgresScanConnector, isKloPostgresConnectionConfig } = await import('@klo/connector-postgres');
if (isKloPostgresConnectionConfig(connection)) {
return new KloPostgresScanConnector({ connectionId, connection });
}
}
if (driver === 'mysql') {
const { KloMysqlScanConnector, isKloMysqlConnectionConfig } = await import('@klo/connector-mysql');
if (isKloMysqlConnectionConfig(connection)) {
return new KloMysqlScanConnector({ connectionId, connection });
}
}
if (driver === 'clickhouse') {
const { KloClickHouseScanConnector, isKloClickHouseConnectionConfig } = await import('@klo/connector-clickhouse');
if (isKloClickHouseConnectionConfig(connection)) {
return new KloClickHouseScanConnector({ connectionId, connection });
}
}
if (driver === 'sqlserver') {
const { KloSqlServerScanConnector, isKloSqlServerConnectionConfig } = await import('@klo/connector-sqlserver');
if (isKloSqlServerConnectionConfig(connection)) {
return new KloSqlServerScanConnector({ connectionId, connection });
}
}
if (driver === 'bigquery') {
const { KloBigQueryScanConnector, isKloBigQueryConnectionConfig } = await import('@klo/connector-bigquery');
if (isKloBigQueryConnectionConfig(connection)) {
const maxBytesBilled = bigQueryMaxBytesBilled(connection);
return new KloBigQueryScanConnector({
connectionId,
connection,
...(maxBytesBilled !== undefined ? { maxBytesBilled } : {}),
});
}
}
if (driver === 'snowflake') {
const { KloSnowflakeScanConnector, isKloSnowflakeConnectionConfig } = await import('@klo/connector-snowflake');
if (isKloSnowflakeConnectionConfig(connection)) {
return new KloSnowflakeScanConnector({ connectionId, connection });
}
}
throw new Error(
`Connection "${connectionId}" uses driver "${driver}", which has no native standalone KLO scan connector. Supported drivers: ${SUPPORTED_DRIVERS}.`,
);
}

View file

@ -0,0 +1,597 @@
/* @jsxImportSource react */
import type { MemoryFlowEvent, MemoryFlowReplayInput } from '@klo/context/ingest/memory-flow';
import { Box, Text } from 'ink';
import React, { type ReactNode } from 'react';
import { buildDemoMetrics, formatCost, formatDuration } from './demo-metrics.js';
import { formatNextStepLines } from './next-steps.js';
import { profileMark } from './startup-profile.js';
profileMark('module:memory-flow-hud');
interface HudTheme {
text: string;
muted: string;
active: string;
complete: string;
warning: string;
failed: string;
border: string;
}
const SPINNER_FRAMES = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'] as const;
function spinner(frame: number): string {
return SPINNER_FRAMES[frame % SPINNER_FRAMES.length] ?? '⠋';
}
function counterValue(target: number, frame: number, framesToFill = 12): number {
if (target <= 0 || frame <= 0) return 0;
if (frame >= framesToFill) return target;
return Math.round((frame / framesToFill) * target);
}
function hasWorkStarted(input: MemoryFlowReplayInput): boolean {
return input.events.some((e) => e.type === 'work_unit_started');
}
function isPrepopulatedDemoReplay(input: MemoryFlowReplayInput): boolean {
return input.metadata?.origin === 'packaged' || input.metadata?.timing === 'prebuilt';
}
function flowLine(width: number, frame: number, active: boolean): string {
if (!active) return '━'.repeat(width);
const pulse = ['░', '▒', '▓', '█', '█', '█', '▓', '▒', '░'];
const pw = pulse.length;
const chars: string[] = [];
const offset = (frame * 2) % (width + pw);
for (let i = 0; i < width; i += 1) {
const p = i - offset + pw;
chars.push(p >= 0 && p < pw ? (pulse[p] ?? '━') : '━');
}
return chars.join('');
}
function brailleFlow(width: number, frame: number): string {
// Braille unicode: U+2800 + dot bitmask
// Dots: 1=0x01 2=0x02 3=0x04 4=0x08 5=0x10 6=0x20 7=0x40 8=0x80
// Layout: col0=[1,2,3,7] col1=[4,5,6,8]
const chars: string[] = [];
for (let i = 0; i < width; i += 1) {
const density = (i + 1) / width;
const phase = (i * 3 + frame * 2) % 12;
let dots = 0;
// Sparse diagonal streams on the left, dense on the right
// Each "stream" is a diagonal line of dots moving rightward
if ((phase + 0) % 4 < density * 4) dots |= 0x01; // dot 1
if ((phase + 1) % 5 < density * 4) dots |= 0x08; // dot 4
if ((phase + 2) % 4 < density * 3) dots |= 0x02; // dot 2
if ((phase + 3) % 5 < density * 3) dots |= 0x10; // dot 5
if ((phase + 4) % 4 < density * 2.5) dots |= 0x04; // dot 3
if ((phase + 5) % 5 < density * 2.5) dots |= 0x20; // dot 6
if ((phase + 1) % 6 < density * 2) dots |= 0x40; // dot 7
if ((phase + 3) % 6 < density * 2) dots |= 0x80; // dot 8
chars.push(String.fromCharCode(0x2800 + dots));
}
return chars.join('');
}
function progressBarOverall(
finishedCount: number,
activeCount: number,
totalCount: number,
width: number,
frame: number,
): string {
if (totalCount === 0) return '░'.repeat(width);
const finishedWidth = Math.round((finishedCount / totalCount) * width);
const activeWidth = Math.max(activeCount > 0 ? 1 : 0, Math.round((activeCount / totalCount) * width));
const queuedWidth = Math.max(0, width - finishedWidth - activeWidth);
const finished = '█'.repeat(finishedWidth);
const pulse = ['░', '▒', '▓', '█', '▓', '▒'];
const pulseLen = pulse.length;
const offset = (frame * 2) % (activeWidth + pulseLen);
const activeChars: string[] = [];
for (let i = 0; i < activeWidth; i += 1) {
const p = i - offset + pulseLen;
activeChars.push(p >= 0 && p < pulseLen ? (pulse[p] ?? '▒') : '▒');
}
return finished + activeChars.join('') + '░'.repeat(queuedWidth);
}
function sparkleWipe(width: number, frame: number, row: number): string {
const chars: string[] = [];
const sweepPos = (frame * 2 + row * 6) % (width + 8);
const sparkles = ['✨', '✦', '✧', '·'];
for (let i = 0; i < width; i += 1) {
const dist = i - sweepPos;
if (dist < -6) {
const t = (i * 11 + row * 5 + frame * 3) % 10;
chars.push(t === 0 ? sparkles[0]! : t === 3 ? sparkles[1]! : t === 7 ? sparkles[2]! : ' ');
} else if (dist < -3) {
const t = (i + frame) % 3;
chars.push(t === 0 ? sparkles[1]! : t === 1 ? sparkles[2]! : sparkles[3]!);
} else if (dist <= 0) {
const gradient = ['░', '▒', '▓', '█'];
chars.push(gradient[Math.min(3, dist + 3)] ?? '█');
} else if (dist <= 2) {
chars.push(dist === 1 ? '▓' : '▒');
} else {
const noise = (i * 31 + row * 17 + frame * 3) % 5;
const messy = ['░', '▒', '▓', '▒', '░'];
chars.push(messy[noise] ?? '▒');
}
}
return chars.join('');
}
function activityWave(width: number, frame: number, offset: number): string {
const heights = ['▁', '▂', '▃', '▄', '▅', '▆', '▇', '█'];
const chars: string[] = [];
for (let i = 0; i < width; i += 1) {
const wave = Math.sin(((i * 2 + frame + offset * 5) * Math.PI) / 6);
const idx = Math.round(((wave + 1) / 2) * (heights.length - 1));
chars.push(heights[idx] ?? '▁');
}
return chars.join('');
}
function topicName(key: string): string {
return (key.split('/').pop()?.replace(/\.md$/, '') ?? key).replace(/[_-]/g, ' ');
}
function tableName(key: string): string {
return key.split('.').pop()?.replace(/[_-]/g, ' ') ?? key;
}
function humanizeInsight(key: string, target: 'sl' | 'wiki', summary: string | undefined): string {
if (summary) return summary;
const name = target === 'sl' ? tableName(key) : topicName(key);
return target === 'sl' ? `Query definition: ${name}` : `Knowledge page: ${name}`;
}
const ADAPTER_PREFIXES = ['live_database_', 'metabase_', 'looker_', 'lookml_', 'metricflow_', 'notion_', 'historic_sql_', 'dbt_descriptions_'];
const INTERNAL_DEMO_CONNECTION_ID = 'orbit_demo';
const PUBLIC_DEMO_SOURCE_LABEL = 'Orbit Demo';
function humanizeUnitKey(unitKey: string): string {
let key = unitKey.replace(/-/g, '_');
for (const prefix of ADAPTER_PREFIXES) {
if (key.startsWith(prefix)) { key = key.slice(prefix.length); break; }
}
return key.replace(/_/g, ' ');
}
interface SourceInfo {
type: string;
name: string;
sourceCount: string;
itemNounPlural: string;
readingVerb: string;
ingestDescription: string;
}
const ADAPTER_LABELS: Record<string, { type: string; plural: string; verb: string; description: string }> = {
'live-database': { type: 'Database', plural: 'tables', verb: 'Reading', description: 'Reading table schemas, understanding relationships, creating query definitions' },
metricflow: { type: 'dbt project', plural: 'models', verb: 'Parsing', description: 'Parsing dbt models, extracting metric definitions, mapping dependencies' },
looker: { type: 'Looker', plural: 'explores', verb: 'Analyzing', description: 'Analyzing explores, extracting dimensions and measures, mapping joins' },
lookml: { type: 'LookML', plural: 'views', verb: 'Parsing', description: 'Parsing LookML views, extracting field definitions, mapping relationships' },
metabase: { type: 'Metabase', plural: 'questions', verb: 'Analyzing', description: 'Analyzing saved questions, extracting query patterns, understanding dashboards' },
notion: { type: 'Notion', plural: 'pages', verb: 'Reading', description: 'Reading pages, extracting structure, understanding your documentation' },
'historic-sql': { type: 'SQL history', plural: 'queries', verb: 'Analyzing', description: 'Analyzing query patterns, identifying common joins, learning access patterns' },
'dbt-descriptions': { type: 'dbt schema', plural: 'models', verb: 'Parsing', description: 'Parsing schema definitions, extracting descriptions, mapping lineage' },
dbt_descriptions: { type: 'dbt', plural: 'models', verb: 'Parsing', description: 'Parsing schema definitions, extracting descriptions, mapping lineage' },
};
function sourceDescription(input: MemoryFlowReplayInput): SourceInfo {
const adapter = input.adapter ?? 'source';
const conn = input.connectionId ?? '';
const sourceEvents = input.events.filter((e) => e.type === 'source_acquired') as Array<{ type: 'source_acquired'; adapter: string; fileCount: number }>;
const isDemoSource = conn === INTERNAL_DEMO_CONNECTION_ID || isPrepopulatedDemoReplay(input);
if (isDemoSource && sourceEvents.length <= 1) {
const count = sourceEvents[0] ? String(sourceEvents[0].fileCount) : '?';
return {
type: PUBLIC_DEMO_SOURCE_LABEL,
name: '',
sourceCount: count,
itemNounPlural: 'sources',
readingVerb: 'Ingesting',
ingestDescription: 'Ingesting warehouse, dbt, BI, and docs into a unified context layer',
};
}
if (sourceEvents.length > 1) {
const totalFiles = sourceEvents.reduce((sum, s) => sum + s.fileCount, 0);
const labels = [...new Set(sourceEvents.map((s) => ADAPTER_LABELS[s.adapter]?.type ?? s.adapter))];
return {
type: labels.join(' + '),
name: conn,
sourceCount: String(totalFiles),
itemNounPlural: 'sources',
readingVerb: 'Ingesting',
ingestDescription: 'Ingesting warehouse, dbt, BI, and docs into a unified context layer',
};
}
const count = sourceEvents[0] ? String(sourceEvents[0].fileCount) : '?';
const info = ADAPTER_LABELS[adapter] ?? { type: adapter, plural: 'sources', verb: 'Reading', description: 'Reading sources, understanding structure, creating definitions' };
return { type: info.type, name: conn, sourceCount: count, itemNounPlural: info.plural, readingVerb: info.verb, ingestDescription: info.description };
}
function activeWorkUnit(
input: MemoryFlowReplayInput,
): { unitKey: string; stepIndex: number; stepBudget: number } | null {
const units = activeWorkUnits(input);
return units.at(-1) ?? null;
}
function activeWorkUnits(
input: MemoryFlowReplayInput,
): Array<{ unitKey: string; stepIndex: number; stepBudget: number }> {
const finishedKeys = new Set<string>();
const unitMap = new Map<string, { stepIndex: number; stepBudget: number }>();
for (const e of input.events) {
if (e.type === 'work_unit_started') {
unitMap.set(e.unitKey, { stepIndex: 0, stepBudget: e.stepBudget });
}
if (e.type === 'work_unit_step') {
const existing = unitMap.get(e.unitKey);
if (existing) {
existing.stepIndex = e.stepIndex;
existing.stepBudget = e.stepBudget;
}
}
if (e.type === 'work_unit_finished') finishedKeys.add(e.unitKey);
}
const result: Array<{ unitKey: string; stepIndex: number; stepBudget: number }> = [];
for (const [unitKey, data] of unitMap) {
if (!finishedKeys.has(unitKey)) result.push({ unitKey, ...data });
}
return result;
}
function queuedWorkUnits(input: MemoryFlowReplayInput): string[] {
const startedKeys = new Set<string>();
for (const e of input.events) {
if (e.type === 'work_unit_started') startedKeys.add(e.unitKey);
}
return input.plannedWorkUnits.filter((u) => !startedKeys.has(u.unitKey)).map((u) => u.unitKey);
}
interface Insight {
icon: string;
text: string;
unitKey: string;
hasSummary: boolean;
}
function buildInsights(input: MemoryFlowReplayInput): Insight[] {
return input.events
.filter((e) => e.type === 'candidate_action')
.map((e) => {
const ca = e as { unitKey: string; target: 'sl' | 'wiki'; key: string };
const detail = input.details.actions.find((a) => a.key === ca.key && a.unitKey === ca.unitKey);
return {
icon: ca.target === 'sl' ? '📊' : '📝',
text: humanizeInsight(ca.key, ca.target, detail?.summary),
unitKey: ca.unitKey,
hasSummary: !!detail?.summary,
};
});
}
function finishedUnits(input: MemoryFlowReplayInput): Array<{ unitKey: string; artifactCount: number }> {
const units: Array<{ unitKey: string; artifactCount: number }> = [];
for (const e of input.events) {
if (e.type === 'work_unit_finished' && e.status === 'success') {
const count = input.events.filter((a) => a.type === 'candidate_action' && a.unitKey === e.unitKey).length;
units.push({ unitKey: e.unitKey, artifactCount: count });
}
}
return units;
}
function artifactCounts(input: MemoryFlowReplayInput): { sl: number; wiki: number } {
let sl = 0;
let wiki = 0;
for (const e of input.events) {
if (e.type === 'candidate_action') {
if (e.target === 'sl') sl++;
else wiki++;
}
}
return { sl, wiki };
}
function pad(str: string, width: number): string {
return str.length >= width ? str : str + ' '.repeat(width - str.length);
}
const KLO_LOGO_SMALL = [
'██╗ ██╗██╗ ██████╗ ',
'██║ ██╔╝██║ ██╔═══██╗',
'█████╔╝ ██║ ██║ ██║',
'██╔═██╗ ██║ ██║ ██║',
'██║ ██╗███████╗╚██████╔╝',
'╚═╝ ╚═╝╚══════╝ ╚═════╝ ',
] as const;
export function Logo(props: { theme: HudTheme; done: boolean }): ReactNode {
const color = props.done ? props.theme.complete : props.theme.active;
return (
<Box flexDirection="column" marginBottom={1} paddingLeft={2}>
{KLO_LOGO_SMALL.map((line, idx) => (
<Text key={idx} color={color}>
{line}
</Text>
))}
</Box>
);
}
export function Hud(props: {
input: MemoryFlowReplayInput;
theme: HudTheme;
frame: number;
width: number;
now?: () => number;
}): ReactNode {
const isRunning = props.input.status === 'running';
const isDone = props.input.status === 'done';
const isFlowing = isRunning && hasWorkStarted(props.input);
const src = sourceDescription(props.input);
const counts = artifactCounts(props.input);
const metrics = buildDemoMetrics(props.input, props.now ? { now: props.now } : {});
const workStarted = hasWorkStarted(props.input);
const sourceEvents = props.input.events.filter((e) => e.type === 'source_acquired');
const col1Content = sourceEvents.length > 1 || !src.name ? src.type : `${src.type} (${src.name})`;
const innerWidth = Math.max(60, props.width - 6);
const actives = activeWorkUnits(props.input);
const reconEvent = props.input.events.find((e) => e.type === 'reconciliation_finished');
const allAnalyzed = isFlowing && actives.length === 0;
const isReconciling = allAnalyzed && !reconEvent && !isDone;
const hLine = '─'.repeat(innerWidth);
const elapsed = formatDuration(metrics.elapsedMs);
let eta = '';
if (metrics.status === 'running' && metrics.etaMs !== null) eta = `~${formatDuration(metrics.etaMs)} left`;
else if (metrics.status !== 'running') eta = 'done';
const cost = workStarted ? formatCost(metrics.estimatedCostUsd) : '';
const statsParts = [`${elapsed}`, eta, cost].filter(Boolean).join(' ');
const prepopulatedCostDisclaimer =
cost && isPrepopulatedDemoReplay(props.input)
? 'Pre-run demo: $ shown is illustrative; no money is being spent now.'
: null;
return (
<Box flexDirection="column" marginBottom={1}>
<Text color={props.theme.border}> {hLine}</Text>
<Text>
<Text color={props.theme.border}> </Text>
<Text color={props.theme.text}>{col1Content}</Text>
<Text color={props.theme.muted}> {src.sourceCount} {src.itemNounPlural}</Text>
</Text>
<Text>
<Text color={props.theme.border}> </Text>
<Text color="#b8860b">{statsParts}</Text>
</Text>
{prepopulatedCostDisclaimer && (
<Text>
<Text color={props.theme.border}> </Text>
<Text color={props.theme.muted}>{prepopulatedCostDisclaimer}</Text>
</Text>
)}
<Text color={props.theme.border}> {hLine}</Text>
</Box>
);
}
export function ActivityFeed(props: {
input: MemoryFlowReplayInput;
theme: HudTheme;
frame: number;
width: number;
completionFrame: number;
showCompletion: boolean;
holdComplete: boolean;
}): ReactNode {
const actives = activeWorkUnits(props.input);
const queued = queuedWorkUnits(props.input);
const finished = finishedUnits(props.input);
const insights = buildInsights(props.input);
const src = sourceDescription(props.input);
const isDone = props.input.status === 'done';
const isError = props.input.status === 'error';
const diffEvent = props.input.events.find((e) => e.type === 'diff_computed') as
| (MemoryFlowEvent & { added: number; modified: number; deleted: number; unchanged: number })
| undefined;
const planEvent = props.input.events.find((e) => e.type === 'chunks_planned') as
| (MemoryFlowEvent & { chunkCount: number; workUnitCount: number })
| undefined;
const reconEvent = props.input.events.find((e) => e.type === 'reconciliation_finished') as
| (MemoryFlowEvent & { conflictCount: number })
| undefined;
const savedEvent = props.input.events.find((e) => e.type === 'saved');
const workStarted = hasWorkStarted(props.input);
const totalChunks = planEvent?.chunkCount ?? 0;
const finishedWithArtifacts = finished.filter((u) => u.artifactCount > 0);
const finishedAreas = totalChunks > 0 ? Math.min(finished.length, totalChunks) : finished.length;
const allWorkDone = workStarted && actives.length === 0 && queued.length === 0;
const isReconciling = allWorkDone && !reconEvent && !isDone && !isError;
const isSaving = reconEvent && !savedEvent && !isDone && !isError;
const isIncremental = diffEvent && (diffEvent.modified > 0 || diffEvent.deleted > 0 || diffEvent.unchanged > 0);
const barWidth = Math.min(40, props.width - 20);
return (
<Box flexDirection="column" paddingLeft={2}>
{/* Phase 1: Connecting */}
{!diffEvent && !workStarted && (
<Text color={props.theme.active}>
{spinner(props.frame)} Connecting to {src.type.toLowerCase()}...
</Text>
)}
{/* Phase 2: Connected */}
{diffEvent && (
<Text color={props.theme.complete}>
Connected found {src.sourceCount} {src.itemNounPlural} to ingest
</Text>
)}
{/* Phase 2b: Diff (incremental runs only) */}
{diffEvent && isIncremental && (
<Text color={props.theme.complete}>
Compared with last sync only re-analyzing what changed
</Text>
)}
{/* Phase 3: Planning */}
{diffEvent && !planEvent && !workStarted && (
<Text color={props.theme.active}>
{spinner(props.frame)} Grouping related {src.itemNounPlural} together for deeper analysis...
</Text>
)}
{planEvent && (
<Text color={props.theme.complete}>
Grouped into {planEvent.chunkCount} business area{planEvent.chunkCount === 1 ? '' : 's'}
</Text>
)}
{/* Phase 4: Ingesting */}
{workStarted && !allWorkDone && (
<Box flexDirection="column" marginTop={1}>
<Text color={props.theme.active}>
{spinner(props.frame)} Ingesting {finishedAreas}/{totalChunks || '?'} business area{totalChunks === 1 ? '' : 's'} done
</Text>
<Text color={props.theme.muted}>
{' '}{src.ingestDescription}
</Text>
{totalChunks > 0 && (
<Text color={props.theme.active}>
{' '}
{progressBarOverall(finishedAreas, actives.length, totalChunks, barWidth, props.frame)}
</Text>
)}
</Box>
)}
{/* Results — what KLO has created */}
{insights.length > 0 && (
<Box flexDirection="column" marginTop={1}>
<Text color={props.theme.text}> Created so far:</Text>
{insights.map((insight, idx) => (
<Text key={`result-${idx}`} color={props.theme.muted}>
{' '}{insight.icon} {insight.text}
</Text>
))}
</Box>
)}
{/* Phase 5: Finalizing */}
{isReconciling && (
<Text color={props.theme.active}>
{spinner(props.frame)} Deduplicating removing overlaps between business areas and checking for conflicts...
</Text>
)}
{reconEvent && (
<Text color={props.theme.complete}>
Deduplicated
{reconEvent.conflictCount > 0
? `${reconEvent.conflictCount} conflict${reconEvent.conflictCount === 1 ? '' : 's'} resolved`
: ' — no conflicts'}
</Text>
)}
{/* Phase 6: Saving */}
{isSaving && (
<Text color={props.theme.active}>{spinner(props.frame)} Saving to context layer...</Text>
)}
{savedEvent && (
<Text color={props.theme.complete}> Saved your agents can now use the KLO context layer</Text>
)}
{/* Phase 7: Completion */}
{props.showCompletion && (isDone || isError) && (
<CompletionSummary input={props.input} theme={props.theme} frame={props.completionFrame} holdComplete={props.holdComplete} />
)}
</Box>
);
}
function CompletionSummary(props: {
input: MemoryFlowReplayInput;
theme: HudTheme;
frame: number;
holdComplete: boolean;
}): ReactNode {
const saved = [...props.input.events].reverse().find((e) => e.type === 'saved');
const wikiCount = saved?.wikiCount ?? 0;
const slCount = saved?.slCount ?? 0;
const isError = props.input.status === 'error';
const sl = counterValue(slCount, props.frame);
const wiki = counterValue(wikiCount, props.frame);
return (
<Box flexDirection="column" marginTop={1}>
{isError ? (
<Text bold color={props.theme.failed}>
Something went wrong review the errors above.
</Text>
) : (
<>
<Text color={props.theme.border}>{'─'.repeat(60)}</Text>
<Text bold color={props.theme.complete}>
KLO finished ingesting your data
</Text>
{(sl > 0 || wiki > 0) && (
<>
<Text />
<Text color={props.theme.text}>KLO created:</Text>
{sl > 0 && (
<Text color={props.theme.active}>
{' '}📊 {sl} query definition{sl === 1 ? '' : 's'} so agents can write accurate SQL for your data
</Text>
)}
{wiki > 0 && (
<Text color={props.theme.complete}>
{' '}📝 {wiki} knowledge page{wiki === 1 ? '' : 's'} so agents understand your business context
</Text>
)}
</>
)}
<Text />
<Text color={props.theme.text}>What to do next:</Text>
{formatNextStepLines().map((line) => (
<Text key={line} color={props.theme.active}>
{line}
</Text>
))}
{props.holdComplete && (
<>
<Text />
<Text color={props.theme.muted}>Press q to exit</Text>
</>
)}
</>
)}
</Box>
);
}

View file

@ -0,0 +1,125 @@
import { EventEmitter } from 'node:events';
import type { MemoryFlowReplayInput } from '@klo/context/ingest';
import { describe, expect, it, vi } from 'vitest';
import { memoryFlowCommandForKey, renderMemoryFlowInteractively } from './memory-flow-interactive.js';
class FakeStdin extends EventEmitter {
isTTY = true;
isRaw = false;
rawModes: boolean[] = [];
resume = vi.fn();
pause = vi.fn();
setRawMode(value: boolean): void {
this.isRaw = value;
this.rawModes.push(value);
}
}
function replay(): MemoryFlowReplayInput {
return {
runId: 'run-1',
connectionId: 'warehouse',
adapter: 'metricflow',
status: 'done',
sourceDir: '/tmp/source',
syncId: 'sync-1',
errors: [],
plannedWorkUnits: [
{
unitKey: 'orders',
rawFiles: ['models/orders.yml'],
peerFileCount: 0,
dependencyCount: 1,
},
{
unitKey: 'customers',
rawFiles: ['models/customers.yml'],
peerFileCount: 0,
dependencyCount: 0,
},
],
details: { actions: [], provenance: [], transcripts: [] },
events: [
{ type: 'source_acquired', adapter: 'metricflow', trigger: 'manual_resync', fileCount: 2 },
{ type: 'scope_detected', fingerprint: null },
{ type: 'raw_snapshot_written', syncId: 'sync-1', rawFileCount: 2 },
{ type: 'diff_computed', added: 1, modified: 1, deleted: 0, unchanged: 0 },
{ type: 'chunks_planned', chunkCount: 2, workUnitCount: 2, evictionCount: 0 },
{ type: 'work_unit_started', unitKey: 'orders', skills: ['knowledge_capture'], stepBudget: 4 },
{ type: 'work_unit_finished', unitKey: 'orders', status: 'success' },
{ type: 'work_unit_started', unitKey: 'customers', skills: ['knowledge_capture'], stepBudget: 4 },
{ type: 'work_unit_finished', unitKey: 'customers', status: 'failed', reason: 'validation reset' },
{ type: 'reconciliation_finished', conflictCount: 0, fallbackCount: 1 },
{ type: 'saved', commitSha: 'abc12345', wikiCount: 1, slCount: 1 },
{ type: 'provenance_recorded', rowCount: 2 },
{ type: 'report_created', runId: 'run-1', reportPath: 'report-1' },
],
};
}
describe('memoryFlowCommandForKey', () => {
it('maps supported terminal key names to memory-flow commands', () => {
const idleSearch = { editing: false, query: '', matchIndex: 0 };
const editingSearch = { editing: true, query: 'c', matchIndex: 0 };
expect(memoryFlowCommandForKey('', idleSearch, { name: 'left' })).toBe('left');
expect(memoryFlowCommandForKey('', idleSearch, { name: 'right' })).toBe('right');
expect(memoryFlowCommandForKey('', idleSearch, { name: 'up' })).toBe('up');
expect(memoryFlowCommandForKey('', idleSearch, { name: 'down' })).toBe('down');
expect(memoryFlowCommandForKey('', idleSearch, { name: 'return' })).toBe('enter');
expect(memoryFlowCommandForKey('', idleSearch, { name: 'tab' })).toBe('tab');
expect(memoryFlowCommandForKey('', idleSearch, { name: 'f' })).toBe('filter');
expect(memoryFlowCommandForKey('', idleSearch, { name: 'p' })).toBe('provenance');
expect(memoryFlowCommandForKey('', idleSearch, { name: 't' })).toBe('transcript');
expect(memoryFlowCommandForKey('', idleSearch, { name: 'q' })).toBe('quit');
expect(memoryFlowCommandForKey('', idleSearch, { name: 'c', ctrl: true })).toBe('quit');
expect(memoryFlowCommandForKey('/', { editing: false, query: '', matchIndex: 0 }, { name: '/' })).toBe(
'search-start',
);
expect(memoryFlowCommandForKey('c', { editing: true, query: '', matchIndex: 0 }, { name: 'c' })).toEqual({
type: 'search-input',
value: 'c',
});
expect(memoryFlowCommandForKey('', editingSearch, { name: 'backspace' })).toBe('search-backspace');
expect(memoryFlowCommandForKey('', editingSearch, { name: 'return' })).toBe('search-submit');
expect(memoryFlowCommandForKey('', editingSearch, { name: 'escape' })).toBe('search-clear');
expect(memoryFlowCommandForKey('', idleSearch, { name: 'x' })).toBeNull();
});
});
describe('renderMemoryFlowInteractively', () => {
it('repaints on keypress and restores raw mode on quit', async () => {
let stdout = '';
const stdin = new FakeStdin();
const prepareKeypressEvents = vi.fn();
const promise = renderMemoryFlowInteractively(
replay(),
{
stdin,
stdout: {
isTTY: true,
columns: 120,
write: (chunk) => {
stdout += chunk;
},
},
},
{ prepareKeypressEvents },
);
stdin.emit('keypress', '', { name: 'right' });
stdin.emit('keypress', '', { name: 'tab' });
stdin.emit('keypress', '', { name: 'q' });
await expect(promise).resolves.toBeUndefined();
expect(prepareKeypressEvents).toHaveBeenCalledWith(stdin);
expect(stdin.rawModes).toEqual([true, false]);
expect(stdin.resume).toHaveBeenCalledTimes(1);
expect(stdin.pause).toHaveBeenCalledTimes(1);
expect(stdout).toContain('\u001b[2J\u001b[H');
expect(stdout).toContain('[ACTIONS]');
expect(stdout).toContain('Pane: trust');
});
});

View file

@ -0,0 +1,143 @@
import { emitKeypressEvents } from 'node:readline';
import {
buildMemoryFlowViewModel,
createInitialMemoryFlowInteractionState,
reduceMemoryFlowInteractionState,
renderMemoryFlowInteractive,
type MemoryFlowInteractionCommand,
type MemoryFlowInteractionState,
type MemoryFlowReplayInput,
} from '@klo/context/ingest';
interface KloMemoryFlowKey {
name?: string;
ctrl?: boolean;
}
export interface KloMemoryFlowStdin {
isTTY?: boolean;
isRaw?: boolean;
setRawMode?(value: boolean): void;
resume?(): void;
pause?(): void;
on(event: 'keypress', listener: (chunk: string, key: KloMemoryFlowKey) => void): this;
off?(event: 'keypress', listener: (chunk: string, key: KloMemoryFlowKey) => void): this;
removeListener?(event: 'keypress', listener: (chunk: string, key: KloMemoryFlowKey) => void): this;
}
interface KloMemoryFlowInteractiveIo {
stdin?: KloMemoryFlowStdin;
stdout: {
isTTY?: boolean;
columns?: number;
write(chunk: string): void;
};
}
interface RenderMemoryFlowInteractiveOptions {
prepareKeypressEvents?(stdin: KloMemoryFlowStdin): void;
}
function defaultPrepareKeypressEvents(stdin: KloMemoryFlowStdin): void {
emitKeypressEvents(stdin as Parameters<typeof emitKeypressEvents>[0]);
}
export function memoryFlowCommandForKey(
chunk: string,
search: MemoryFlowInteractionState['search'],
key: KloMemoryFlowKey,
): MemoryFlowInteractionCommand | null {
if (search.editing) {
if (key.name === 'escape') return 'search-clear';
if (key.name === 'return' || key.name === 'enter') return 'search-submit';
if (key.name === 'backspace') return 'search-backspace';
if (chunk.length === 1 && chunk >= ' ' && chunk !== '\u007f') {
return { type: 'search-input', value: chunk };
}
return null;
}
if (key.ctrl === true && key.name === 'c') {
return 'quit';
}
if (key.name === '/') return 'search-start';
if (key.name === 'left') return 'left';
if (key.name === 'right') return 'right';
if (key.name === 'up') return 'up';
if (key.name === 'down') return 'down';
if (key.name === 'return' || key.name === 'enter') return 'enter';
if (key.name === 'tab') return 'tab';
if (key.name === 'f') return 'filter';
if (key.name === 'p') return 'provenance';
if (key.name === 't') return 'transcript';
if (key.name === 'q' || key.name === 'escape') return 'quit';
return null;
}
function removeKeypressListener(
stdin: KloMemoryFlowStdin,
handler: (chunk: string, key: KloMemoryFlowKey) => void,
): void {
if (stdin.off) {
stdin.off('keypress', handler);
return;
}
stdin.removeListener?.('keypress', handler);
}
function repaint(input: MemoryFlowReplayInput, state: MemoryFlowInteractionState, io: KloMemoryFlowInteractiveIo): void {
const view = buildMemoryFlowViewModel(input);
io.stdout.write('\u001b[2J\u001b[H');
io.stdout.write(renderMemoryFlowInteractive(view, state, { terminalWidth: io.stdout.columns }));
}
export async function renderMemoryFlowInteractively(
input: MemoryFlowReplayInput,
io: KloMemoryFlowInteractiveIo,
options: RenderMemoryFlowInteractiveOptions = {},
): Promise<void> {
const stdin = io.stdin;
if (stdin?.isTTY !== true) {
const view = buildMemoryFlowViewModel(input);
io.stdout.write(
renderMemoryFlowInteractive(view, createInitialMemoryFlowInteractionState(view), {
terminalWidth: io.stdout.columns,
}),
);
return;
}
const view = buildMemoryFlowViewModel(input);
let state = createInitialMemoryFlowInteractionState(view);
const previousRawMode = stdin.isRaw === true;
return new Promise((resolve) => {
const cleanup = (): void => {
removeKeypressListener(stdin, handleKeypress);
stdin.setRawMode?.(previousRawMode);
stdin.pause?.();
};
const handleKeypress = (_chunk: string, key: KloMemoryFlowKey): void => {
const command = memoryFlowCommandForKey(_chunk, state.search, key);
if (!command) {
return;
}
state = reduceMemoryFlowInteractionState(state, command, view);
repaint(input, state, io);
if (state.shouldQuit) {
cleanup();
resolve();
}
};
(options.prepareKeypressEvents ?? defaultPrepareKeypressEvents)(stdin);
stdin.setRawMode?.(true);
stdin.resume?.();
stdin.on('keypress', handleKeypress);
repaint(input, state, io);
});
}

View file

@ -0,0 +1,315 @@
/* @jsxImportSource react */
import type { MemoryFlowReplayInput } from '@klo/context/ingest';
import { render as renderInkTest } from 'ink-testing-library';
import React, { type ReactNode } from 'react';
import { describe, expect, it, vi } from 'vitest';
import {
MemoryFlowTuiApp,
memoryFlowCommandForInkInput,
renderMemoryFlowTui,
sanitizeMemoryFlowTuiError,
startLiveMemoryFlowTui,
type KloMemoryFlowTuiIo,
type MemoryFlowInkInstance,
type MemoryFlowInkRenderOptions,
} from './memory-flow-tui.js';
function replayInput(): MemoryFlowReplayInput {
return {
runId: 'run-1', connectionId: 'warehouse', adapter: 'live-database',
status: 'done', sourceDir: null, syncId: 'sync-1', reportId: 'report-1', reportPath: 'report-1', errors: [],
plannedWorkUnits: [
{ unitKey: 'orders', rawFiles: ['orders'], peerFileCount: 0, dependencyCount: 1 },
{ unitKey: 'customers', rawFiles: ['customers'], peerFileCount: 1, dependencyCount: 0 },
],
details: {
actions: [
{ unitKey: 'orders', target: 'wiki', action: 'created', key: 'knowledge/orders.md', summary: 'order lifecycle', rawFiles: ['orders'], status: 'success' },
{ unitKey: 'customers', target: 'sl', action: 'updated', key: 'orbit_demo.customers', summary: 'customer metrics', rawFiles: ['customers'], status: 'success' },
],
provenance: [{ rawPath: 'orders', artifactKind: 'wiki', artifactKey: 'knowledge/orders.md', actionType: 'wiki_written' }],
transcripts: [{ unitKey: 'orders', path: '/tmp/t.jsonl', toolCallCount: 2, errorCount: 0, toolNames: ['read_raw_span', 'wiki_write'] }],
},
events: [
{ type: 'source_acquired', adapter: 'live-database', trigger: 'manual_resync', fileCount: 2 },
{ type: 'scope_detected', fingerprint: 'scope-1' },
{ type: 'raw_snapshot_written', syncId: 'sync-1', rawFileCount: 2 },
{ type: 'diff_computed', added: 1, modified: 1, deleted: 0, unchanged: 0 },
{ type: 'chunks_planned', chunkCount: 2, workUnitCount: 2, evictionCount: 0 },
{ type: 'work_unit_started', unitKey: 'orders', skills: ['knowledge_capture'], stepBudget: 40 },
{ type: 'candidate_action', unitKey: 'orders', target: 'wiki', action: 'created', key: 'knowledge/orders.md' },
{ type: 'work_unit_finished', unitKey: 'orders', status: 'success' },
{ type: 'work_unit_started', unitKey: 'customers', skills: ['sl_capture'], stepBudget: 40 },
{ type: 'candidate_action', unitKey: 'customers', target: 'sl', action: 'updated', key: 'orbit_demo.customers' },
{ type: 'work_unit_finished', unitKey: 'customers', status: 'success' },
{ type: 'reconciliation_finished', conflictCount: 0, fallbackCount: 0 },
{ type: 'saved', commitSha: 'commit-one', wikiCount: 1, slCount: 1 },
{ type: 'provenance_recorded', rowCount: 1 },
{ type: 'report_created', runId: 'run-1', reportPath: 'report-1' },
],
};
}
function runningReplayInput(): MemoryFlowReplayInput {
return { ...replayInput(), status: 'running', syncId: 'pending', reportId: undefined, reportPath: undefined, plannedWorkUnits: [], events: [{ type: 'source_acquired', adapter: 'live-database', trigger: 'manual_resync', fileCount: 1 }] };
}
function packagedReplayInput(overrides: Partial<MemoryFlowReplayInput> = {}): MemoryFlowReplayInput {
return {
...replayInput(),
connectionId: 'orbit_demo',
metadata: {
schemaVersion: 1,
mode: 'seeded',
origin: 'packaged',
timing: 'prebuilt',
capturedAt: null,
sourceReportId: 'demo-seeded-report',
sourceReportPath: 'reports/seeded-demo-report.json',
fallbackReason: null,
},
...overrides,
};
}
function makeIo(): { io: KloMemoryFlowTuiIo; stderr: () => string } {
let stderr = '';
return { io: { stdin: { isTTY: true, setRawMode: vi.fn() }, stdout: { isTTY: true, columns: 120, write: vi.fn() }, stderr: { write(chunk: string) { stderr += chunk; } } }, stderr: () => stderr };
}
function fakeInkInstance(): MemoryFlowInkInstance {
return { rerender: vi.fn(), unmount: vi.fn(), waitUntilExit: vi.fn(async () => undefined), clear: vi.fn() };
}
async function waitForInkInput(): Promise<void> { await new Promise((r) => setTimeout(r, 10)); }
function renderedAppProps(tree: ReactNode): Record<string, unknown> {
expect(React.isValidElement(tree)).toBe(true);
return (tree as React.ReactElement<Record<string, unknown>>).props;
}
describe('memoryFlowCommandForInkInput', () => {
it('maps input to commands', () => {
expect(memoryFlowCommandForInkInput('q', {})).toBe('quit');
expect(memoryFlowCommandForInkInput('c', { ctrl: true })).toBe('quit');
expect(memoryFlowCommandForInkInput('x', {})).toBeNull();
});
});
describe('sanitizeMemoryFlowTuiError', () => {
it('redacts credentials', () => {
expect(sanitizeMemoryFlowTuiError(new Error('postgres://x?api_key=y password=z'))).toBe('[redacted-url] [redacted]');
});
});
describe('MemoryFlowTuiApp', () => {
it('always shows the KLO logo', () => {
const { lastFrame } = renderInkTest(<MemoryFlowTuiApp input={replayInput()} terminalWidth={120} onExit={vi.fn()} showBoot={false} />);
expect(lastFrame()).toContain('█████╔╝');
});
it('shows persistent HUD with source and status terminology', () => {
const { lastFrame } = renderInkTest(<MemoryFlowTuiApp input={{ ...replayInput(), connectionId: 'warehouse' }} terminalWidth={120} onExit={vi.fn()} showBoot={false} />);
const frame = lastFrame() ?? '';
expect(frame).toContain('Database (warehouse)');
expect(frame).toContain('2 tables');
expect(frame).toContain('done');
expect(frame).toContain('warehouse');
expect(frame).toContain('╭');
expect(frame).toContain('╰');
});
it('hides the internal demo connection id before packaged replay source events are visible', () => {
const { lastFrame } = renderInkTest(
<MemoryFlowTuiApp
input={packagedReplayInput({ status: 'running', events: [] })}
terminalWidth={120}
onExit={vi.fn()}
showBoot={false}
/>,
);
const frame = lastFrame() ?? '';
expect(frame).toContain('Orbit Demo');
expect(frame).not.toContain('orbit_demo');
expect(frame).not.toContain('Database (orbit_demo)');
});
it('keeps the packaged replay source label public while only one source event is visible', () => {
const { lastFrame } = renderInkTest(
<MemoryFlowTuiApp
input={packagedReplayInput({
status: 'running',
events: [{ type: 'source_acquired', adapter: 'live-database', trigger: 'demo_seeded', fileCount: 8 }],
})}
terminalWidth={120}
onExit={vi.fn()}
showBoot={false}
/>,
);
const frame = lastFrame() ?? '';
expect(frame).toContain('Orbit Demo');
expect(frame).not.toContain('orbit_demo');
expect(frame).not.toContain('Database (orbit_demo)');
});
it('shows a prepopulated data disclaimer for packaged demo replay cost estimates', () => {
const { lastFrame } = renderInkTest(
<MemoryFlowTuiApp
input={packagedReplayInput()}
terminalWidth={120}
onExit={vi.fn()}
showBoot={false}
/>,
);
const frame = lastFrame() ?? '';
expect(frame).toContain('$');
expect(frame).toContain('Pre-run demo: $ shown is illustrative; no money is being spent now.');
expect(frame).not.toContain('orbit_demo');
});
it('does not show the prepopulated data disclaimer for captured full replay cost estimates', () => {
const { lastFrame } = renderInkTest(
<MemoryFlowTuiApp
input={{
...replayInput(),
metadata: {
schemaVersion: 1,
mode: 'full',
origin: 'captured',
timing: 'captured',
capturedAt: '2026-05-01T00:00:00.000Z',
sourceReportId: 'report-full',
sourceReportPath: 'reports/report-full.json',
fallbackReason: null,
},
}}
terminalWidth={120}
onExit={vi.fn()}
showBoot={false}
/>,
);
expect(lastFrame()).not.toContain('Demo data is prepopulated');
});
it('shows accumulated activity feed on completion', () => {
const { lastFrame } = renderInkTest(<MemoryFlowTuiApp input={replayInput()} terminalWidth={120} onExit={vi.fn()} showBoot={false} />);
const frame = lastFrame() ?? '';
expect(frame).toContain('Connected — found 2 tables to ingest');
expect(frame).toContain('Created so far:');
expect(frame).toContain('order lifecycle');
expect(frame).toContain('customer metrics');
expect(frame).toContain('KLO finished ingesting your data');
expect(frame).toContain('klo sl list');
expect(frame).toContain('klo wiki list');
expect(frame).toContain('klo serve --mcp stdio --user-id local');
expect(frame).not.toContain(['klo', 'ask'].join(' '));
expect(frame).not.toContain(['klo', 'mcp'].join(' '));
});
it('handles quit while running', async () => {
const onExit = vi.fn();
const { stdin } = renderInkTest(<MemoryFlowTuiApp input={runningReplayInput()} terminalWidth={120} onExit={onExit} showBoot={false} />);
stdin.write('q');
await waitForInkInput();
expect(onExit).toHaveBeenCalledTimes(1);
});
it('shows active work unit with progress', () => {
const running: MemoryFlowReplayInput = {
...runningReplayInput(),
events: [
{ type: 'source_acquired', adapter: 'live-database', trigger: 'manual_resync', fileCount: 1 },
{ type: 'diff_computed', added: 1, modified: 0, deleted: 0, unchanged: 0 },
{ type: 'chunks_planned', chunkCount: 1, workUnitCount: 1, evictionCount: 0 },
{ type: 'work_unit_started', unitKey: 'orders', skills: ['knowledge_capture'], stepBudget: 40 },
],
plannedWorkUnits: [{ unitKey: 'orders', rawFiles: ['orders'], peerFileCount: 0, dependencyCount: 1 }],
};
const { lastFrame } = renderInkTest(<MemoryFlowTuiApp input={running} terminalWidth={120} onExit={vi.fn()} showBoot={false} />);
const frame = lastFrame() ?? '';
expect(frame).toContain('Ingesting — 0/1 business area done');
expect(frame).toContain('Reading table schemas, understanding relationships, creating query definitions');
expect(frame).toContain('█████╔╝');
});
it('describes multi-source ingestion as building the context layer', () => {
const running: MemoryFlowReplayInput = {
...runningReplayInput(),
adapter: 'multi-source',
events: [
{ type: 'source_acquired', adapter: 'live-database', trigger: 'manual_resync', fileCount: 8 },
{ type: 'source_acquired', adapter: 'dbt-descriptions', trigger: 'manual_resync', fileCount: 3 },
{ type: 'diff_computed', added: 11, modified: 0, deleted: 0, unchanged: 0 },
{ type: 'chunks_planned', chunkCount: 1, workUnitCount: 1, evictionCount: 0 },
{ type: 'work_unit_started', unitKey: 'orders', skills: ['knowledge_capture'], stepBudget: 40 },
],
plannedWorkUnits: [{ unitKey: 'orders', rawFiles: ['orders'], peerFileCount: 0, dependencyCount: 1 }],
};
const { lastFrame } = renderInkTest(<MemoryFlowTuiApp input={running} terminalWidth={120} onExit={vi.fn()} showBoot={false} />);
const frame = lastFrame() ?? '';
expect(frame).toContain('Ingesting warehouse, dbt, BI, and docs into a unified context layer');
expect(frame).not.toContain('unified semantic layer');
});
it('hides completion while running', () => {
const { lastFrame } = renderInkTest(<MemoryFlowTuiApp input={runningReplayInput()} terminalWidth={120} onExit={vi.fn()} showBoot={false} />);
expect(lastFrame()).not.toContain('KLO finished ingesting');
});
});
describe('startLiveMemoryFlowTui', () => {
it('starts and updates', async () => {
const { io } = makeIo();
const instance = fakeInkInstance();
const live = await startLiveMemoryFlowTui(runningReplayInput(), io, { renderInk: () => instance });
expect(live).not.toBeNull();
live?.update(replayInput());
expect(instance.rerender).toHaveBeenCalledTimes(1);
live?.close();
expect(instance.unmount).toHaveBeenCalledTimes(1);
});
it('redacts errors', async () => {
const { io, stderr } = makeIo();
await expect(startLiveMemoryFlowTui(runningReplayInput(), io, { renderInk: () => { throw new Error('postgres://x?token=y'); } })).resolves.toBeNull();
expect(stderr()).toContain('[redacted-url]');
});
});
describe('renderMemoryFlowTui', () => {
it('renders and returns true', async () => {
const { io } = makeIo();
const instance = fakeInkInstance();
await expect(renderMemoryFlowTui(replayInput(), io, { renderInk: () => instance })).resolves.toBe(true);
});
it('scales event timing with the speed multiplier while keeping animations normal speed', async () => {
const { io } = makeIo();
const instance = fakeInkInstance();
let renderedTree: ReactNode = null;
await expect(
renderMemoryFlowTui(replayInput(), io, {
speedMultiplier: 0.125,
renderInk: (tree) => {
renderedTree = tree;
return instance;
},
}),
).resolves.toBe(true);
expect(renderedAppProps(renderedTree)).toMatchObject({
paceMsPerEvent: 1440,
frameMs: 140,
completionFrameMs: 80,
completionHoldMs: 1000,
});
});
it('redacts errors', async () => {
const { io, stderr } = makeIo();
await expect(renderMemoryFlowTui(replayInput(), io, { renderInk: () => { throw new Error('postgres://x?token=y'); } })).resolves.toBe(false);
expect(stderr()).toContain('[redacted-url]');
});
});

View file

@ -0,0 +1,552 @@
/* @jsxImportSource react */
import {
buildMemoryFlowViewModel,
buildMemoryFlowVisualModel,
createInitialMemoryFlowInteractionState,
findMemoryFlowSearchMatches,
type MemoryFlowColumnId,
type MemoryFlowInteractionCommand,
type MemoryFlowInteractionState,
type MemoryFlowReplayInput,
type MemoryFlowViewModel,
reduceMemoryFlowInteractionState,
selectedMemoryFlowColumn,
selectedMemoryFlowDetails,
} from '@klo/context/ingest';
import { Box, Text, render as renderInkRuntime, useApp, useInput } from 'ink';
import React, { type ReactNode, useEffect, useMemo, useRef, useState } from 'react';
import { buildDemoMetrics } from './demo-metrics.js';
import {
ActivityFeed,
Hud,
Logo,
} from './memory-flow-hud.js';
import { profileMark } from './startup-profile.js';
profileMark('module:memory-flow-tui');
const COLOR_THEME = {
text: 'white',
muted: 'gray',
active: 'cyan',
complete: 'green',
warning: 'yellow',
failed: 'red',
border: 'gray',
} as const;
const NO_COLOR_THEME = {
text: 'white',
muted: 'white',
active: 'white',
complete: 'white',
warning: 'white',
failed: 'white',
border: 'white',
} as const;
type MemoryFlowTuiTheme = Record<keyof typeof COLOR_THEME, string>;
const STAGE_LABELS = {
source: 'CONNECT',
chunks: 'SNAPSHOT',
workUnits: 'PLAN',
actions: 'ANALYZE',
gates: 'VALIDATE',
saved: 'MEMORY',
} satisfies Record<MemoryFlowColumnId, string>;
export interface KloMemoryFlowTuiIo {
stdin?: { isTTY?: boolean; setRawMode?(value: boolean): void };
stdout: { isTTY?: boolean; columns?: number; write(chunk: string): void };
stderr: { write(chunk: string): void };
}
export interface MemoryFlowTuiLiveSession {
update(input: MemoryFlowReplayInput): void;
close(): void;
isClosed(): boolean;
}
export interface MemoryFlowInkInstance {
rerender(tree: ReactNode): void;
unmount(): void;
waitUntilExit(): Promise<void>;
clear?(): void;
}
export interface MemoryFlowInkRenderOptions {
stdin?: KloMemoryFlowTuiIo['stdin'];
stdout: KloMemoryFlowTuiIo['stdout'];
stderr: KloMemoryFlowTuiIo['stderr'];
exitOnCtrlC: boolean;
patchConsole: boolean;
maxFps: number;
alternateScreen: boolean;
}
interface RenderMemoryFlowTuiOptions {
renderInk?: (tree: ReactNode, options: MemoryFlowInkRenderOptions) => MemoryFlowInkInstance;
paceEvents?: boolean;
paceMsPerEvent?: number;
speedMultiplier?: number;
}
interface StartLiveMemoryFlowTuiOptions {
renderInk?: (tree: ReactNode, options: MemoryFlowInkRenderOptions) => MemoryFlowInkInstance;
}
interface RenderTreeOptions {
paceEvents?: boolean;
paceMsPerEvent?: number;
frameMs?: number;
completionFrameMs?: number;
completionHoldMs?: number;
}
interface MemoryFlowTuiTiming {
paceMsPerEvent: number;
frameMs: number;
completionFrameMs: number;
completionHoldMs: number;
}
const DEFAULT_TUI_TIMING = {
paceMsPerEvent: 180,
frameMs: 140,
completionFrameMs: 80,
completionHoldMs: 1000,
} satisfies MemoryFlowTuiTiming;
interface InkKey {
leftArrow?: boolean;
rightArrow?: boolean;
upArrow?: boolean;
downArrow?: boolean;
return?: boolean;
escape?: boolean;
ctrl?: boolean;
shift?: boolean;
tab?: boolean;
backspace?: boolean;
delete?: boolean;
}
interface MemoryFlowTuiAppProps {
input: MemoryFlowReplayInput;
terminalWidth?: number;
env?: NodeJS.ProcessEnv;
onExit(): void;
paceEvents?: boolean;
paceMsPerEvent?: number;
frameMs?: number;
completionFrameMs?: number;
completionHoldMs?: number;
showBoot?: boolean;
}
function resolveMemoryFlowTuiTheme(env: NodeJS.ProcessEnv = process.env): MemoryFlowTuiTheme {
if (env.NO_COLOR || env.TERM === 'dumb') {
return NO_COLOR_THEME;
}
return COLOR_THEME;
}
export function sanitizeMemoryFlowTuiError(error: unknown): string {
const message = error instanceof Error ? error.message : String(error);
return message
.replace(/[a-z][a-z0-9+.-]*:\/\/[^\s]+/gi, '[redacted-url]')
.replace(/\b(api[_-]?key|password|token|secret)=\S+/gi, '[redacted]');
}
export function memoryFlowCommandForInkInput(
input: string,
key: InkKey,
search: MemoryFlowInteractionState['search'] = { editing: false, query: '', matchIndex: 0 },
): MemoryFlowInteractionCommand | null {
if (search.editing) {
if (key.escape) return 'search-clear';
if (key.return) return 'search-submit';
if (key.backspace || key.delete) return 'search-backspace';
if (key.downArrow || (key.tab && !key.shift)) return 'search-next';
if (key.upArrow || (key.tab && key.shift)) return 'search-previous';
if (input.length === 1 && input >= ' ' && input !== '') {
return { type: 'search-input', value: input };
}
return null;
}
if (key.ctrl === true && input === 'c') return 'quit';
if (input === '/') return 'search-start';
if (search.query && input === 'n') return 'search-next';
if (search.query && input === 'N') return 'search-previous';
if (input === '') return 'left';
if (input === '') return 'right';
if (input === '') return 'up';
if (input === '') return 'down';
if (key.leftArrow) return 'left';
if (key.rightArrow) return 'right';
if (key.upArrow) return 'up';
if (key.downArrow) return 'down';
if (key.return) return 'enter';
if (key.tab) return 'tab';
if (input === 'f') return 'filter';
if (input === 'p') return 'provenance';
if (input === 't') return 'transcript';
if (input === 'q' || key.escape) return 'quit';
return null;
}
function stageLabel(columnId: MemoryFlowColumnId): string {
return STAGE_LABELS[columnId];
}
function statusLabel(status: string): 'OK' | 'RUN' | 'WARN' | 'FAIL' | 'WAIT' {
if (status === 'complete') return 'OK';
if (status === 'active') return 'RUN';
if (status === 'warning') return 'WARN';
if (status === 'failed') return 'FAIL';
return 'WAIT';
}
function filterLabel(filter: MemoryFlowInteractionState['filter']): string {
return filter === 'failed_or_flagged' ? 'issues' : 'all';
}
function searchStatusLine(view: MemoryFlowViewModel, state: MemoryFlowInteractionState): string | null {
if (!state.search.editing && state.search.query.length === 0) {
return null;
}
const matches = findMemoryFlowSearchMatches(view, state.search.query);
const status = state.search.editing ? 'editing' : 'locked';
const position = matches.length === 0 ? '0/0' : `${state.search.matchIndex + 1}/${matches.length}`;
return `Search: ${state.search.query || '/'} (${position} matches, ${status})`;
}
function humanizeDemoText(value: string): string {
return value
.replace(/\bWORKUNITS\b/g, 'PLAN')
.replace(/\bWorkUnit\b/g, 'Table review')
.replace(/\bwork units\b/gi, 'table reviews')
.replace(/\bwork-unit\b/gi, 'table-review')
.replace(/\bWUs\b/g, 'tables')
.replace(/\bchunks\b/gi, 'table groups')
.replace(/\bcandidates\b/gi, 'drafts')
.replace(/\bcandidate\b/gi, 'draft')
.replace(/\braw files\b/gi, 'database files')
.replace(/\braw file\b/gi, 'database file')
.replace(/\bSL\b/g, 'context layer');
}
function DetailsPane(props: {
view: MemoryFlowViewModel;
state: MemoryFlowInteractionState;
theme: MemoryFlowTuiTheme;
}): ReactNode {
const column = selectedMemoryFlowColumn(props.view, props.state);
const details = selectedMemoryFlowDetails(props.view, props.state).map(humanizeDemoText).slice(0, 8);
const rawFiles = Array.from(
new Set([
...props.view.details.actions.flatMap((action) => action.rawFiles),
...props.view.details.provenance.map((row) => row.rawPath),
]),
).slice(0, 4);
const searchLine = searchStatusLine(props.view, props.state);
return (
<Box flexDirection="column">
<Text color={props.theme.active}>
Details / focus: {stageLabel(column.id)} Pane: {props.state.pane} Filter: {filterLabel(props.state.filter)}
</Text>
{searchLine && <Text color={props.theme.active}>{searchLine}</Text>}
{details.map((detail, index) => (
<Text key={`${index}-${detail}`} color={props.theme.text}>
- {detail}
</Text>
))}
{rawFiles.map((rawFile) => (
<Text key={rawFile} color={props.theme.muted}>
- {rawFile}
</Text>
))}
{props.view.completionLine && <Text color={props.theme.complete}>{humanizeDemoText(props.view.completionLine)}</Text>}
</Box>
);
}
function TrustIssues(props: { view: MemoryFlowViewModel; theme: MemoryFlowTuiTheme }): ReactNode {
if (props.view.trustIssues.length === 0) {
return null;
}
return (
<Box flexDirection="column" marginBottom={1}>
<Text color={props.theme.warning}>Validation notes</Text>
{props.view.trustIssues.slice(0, 4).map((issue) => (
<Text
key={`${issue.severity}-${issue.title}`}
color={issue.severity === 'failed' ? props.theme.failed : props.theme.warning}
>
{issue.severity === 'failed' ? 'FAILED' : 'WARNING'} {humanizeDemoText(issue.title)}:{' '}
{humanizeDemoText(issue.detail)}
</Text>
))}
</Box>
);
}
export function MemoryFlowTuiApp(props: MemoryFlowTuiAppProps): ReactNode {
const app = useApp();
const totalEvents = props.input.events.length;
const paceEnabled = props.paceEvents === true && totalEvents > 0;
const [pacedCount, setPacedCount] = useState<number>(paceEnabled ? 0 : totalEvents);
const pacedInput = useMemo<MemoryFlowReplayInput>(() => {
if (!paceEnabled || pacedCount >= totalEvents) {
return props.input;
}
return {
...props.input,
status: 'running',
events: props.input.events.slice(0, pacedCount),
};
}, [paceEnabled, pacedCount, totalEvents, props.input]);
const pacedNow = useMemo<(() => number) | undefined>(() => {
if (!paceEnabled) return undefined;
const firstEvent = props.input.events[0];
if (!firstEvent?.emittedAt) return undefined;
const firstEventMs = Date.parse(firstEvent.emittedAt);
if (!Number.isFinite(firstEventMs)) return undefined;
const stride = props.paceMsPerEvent ?? DEFAULT_TUI_TIMING.paceMsPerEvent;
return () => firstEventMs + pacedCount * stride;
}, [paceEnabled, pacedCount, props.input.events, props.paceMsPerEvent]);
const view = useMemo(() => buildMemoryFlowViewModel(pacedInput), [pacedInput]);
const [state, setState] = useState<MemoryFlowInteractionState>(() => createInitialMemoryFlowInteractionState(view));
const [frame, setFrame] = useState(0);
const [thoughtFrame, setThoughtFrame] = useState(0);
const [completionFrame, setCompletionFrame] = useState(0);
const [holdComplete, setHoldComplete] = useState(false);
const [userHasNavigated, setUserHasNavigated] = useState(false);
const lastEventCountRef = useRef(pacedInput.events.length);
const lastStatusRef = useRef(pacedInput.status);
const exitHandled = useRef(false);
const theme = resolveMemoryFlowTuiTheme(props.env);
useEffect(() => {
if (!state.shouldQuit || exitHandled.current) {
return;
}
exitHandled.current = true;
props.onExit();
app.exit();
}, [app, props, state.shouldQuit]);
useEffect(() => {
const timer = setInterval(() => {
setFrame((current) => current + 1);
setThoughtFrame((current) => current + 1);
}, props.frameMs ?? DEFAULT_TUI_TIMING.frameMs);
return () => clearInterval(timer);
}, [props.frameMs]);
useEffect(() => {
if (lastEventCountRef.current !== pacedInput.events.length) {
lastEventCountRef.current = pacedInput.events.length;
setThoughtFrame(0);
}
}, [pacedInput.events.length]);
useEffect(() => {
if (lastStatusRef.current !== pacedInput.status) {
lastStatusRef.current = pacedInput.status;
if (pacedInput.status === 'done' || pacedInput.status === 'error') {
setCompletionFrame(0);
}
}
}, [pacedInput.status]);
useEffect(() => {
if (pacedInput.status !== 'done' && pacedInput.status !== 'error') return;
if (completionFrame >= 12) return;
const timer = setInterval(
() => setCompletionFrame((current) => Math.min(12, current + 1)),
props.completionFrameMs ?? DEFAULT_TUI_TIMING.completionFrameMs,
);
return () => clearInterval(timer);
}, [pacedInput.status, completionFrame, props.completionFrameMs]);
useEffect(() => {
if (completionFrame < 12) {
setHoldComplete(false);
return;
}
const timer = setTimeout(
() => setHoldComplete(true),
props.completionHoldMs ?? DEFAULT_TUI_TIMING.completionHoldMs,
);
return () => clearTimeout(timer);
}, [completionFrame, props.completionHoldMs]);
useEffect(() => {
if (!paceEnabled || pacedCount >= totalEvents) {
return;
}
const interval = props.paceMsPerEvent ?? DEFAULT_TUI_TIMING.paceMsPerEvent;
const timer = setInterval(() => {
setPacedCount((current) => Math.min(totalEvents, current + 1));
}, interval);
return () => clearInterval(timer);
}, [paceEnabled, pacedCount, totalEvents, props.paceMsPerEvent]);
useInput((input, key) => {
const command = memoryFlowCommandForInkInput(input, key, state.search);
if (!command) return;
if (command === 'quit' && isComplete && !holdComplete) return;
if (command !== 'quit') setUserHasNavigated(true);
setState((current) => reduceMemoryFlowInteractionState(current, command, view));
});
const isComplete = pacedInput.status === 'done' || pacedInput.status === 'error';
const completionMetrics = useMemo(
() => buildDemoMetrics(pacedInput, pacedNow ? { now: pacedNow } : {}),
[pacedInput, pacedNow],
);
const termWidth = props.terminalWidth ?? 80;
return (
<Box flexDirection="column" paddingX={1}>
<Logo theme={theme} done={isComplete} />
<Hud input={pacedInput} theme={theme} frame={frame} width={termWidth} now={pacedNow} />
<ActivityFeed input={pacedInput} theme={theme} frame={frame} width={termWidth} completionFrame={completionFrame} showCompletion={isComplete} holdComplete={holdComplete} />
<TrustIssues view={view} theme={theme} />
{userHasNavigated && <DetailsPane view={view} state={state} theme={theme} />}
</Box>
);
}
function renderTree(
input: MemoryFlowReplayInput,
io: KloMemoryFlowTuiIo,
onExit: () => void,
options: RenderTreeOptions = {},
): ReactNode {
return (
<MemoryFlowTuiApp
input={input}
terminalWidth={io.stdout.columns ?? process.stdout.columns}
onExit={onExit}
paceEvents={options.paceEvents}
paceMsPerEvent={options.paceMsPerEvent}
frameMs={options.frameMs}
completionFrameMs={options.completionFrameMs}
completionHoldMs={options.completionHoldMs}
/>
);
}
function renderInk(tree: ReactNode, options: MemoryFlowInkRenderOptions): MemoryFlowInkInstance {
return renderInkRuntime(tree, {
stdin: options.stdin as NodeJS.ReadStream | undefined,
stdout: options.stdout as NodeJS.WriteStream,
stderr: options.stderr as NodeJS.WriteStream,
exitOnCtrlC: options.exitOnCtrlC,
patchConsole: options.patchConsole,
maxFps: options.maxFps,
alternateScreen: options.alternateScreen,
}) as MemoryFlowInkInstance;
}
function renderOptions(io: KloMemoryFlowTuiIo): MemoryFlowInkRenderOptions {
return {
stdin: io.stdin,
stdout: io.stdout,
stderr: io.stderr,
exitOnCtrlC: false,
patchConsole: false,
maxFps: 30,
alternateScreen: true,
};
}
function scaleTiming(ms: number, speedMultiplier: number): number {
return Math.max(20, Math.round(ms / speedMultiplier));
}
function resolveTiming(options: RenderMemoryFlowTuiOptions): MemoryFlowTuiTiming {
const speedMultiplier =
typeof options.speedMultiplier === 'number' && options.speedMultiplier > 0 ? options.speedMultiplier : 1;
return {
paceMsPerEvent:
typeof options.paceMsPerEvent === 'number' && options.paceMsPerEvent > 0
? options.paceMsPerEvent
: scaleTiming(DEFAULT_TUI_TIMING.paceMsPerEvent, speedMultiplier),
frameMs: DEFAULT_TUI_TIMING.frameMs,
completionFrameMs: DEFAULT_TUI_TIMING.completionFrameMs,
completionHoldMs: DEFAULT_TUI_TIMING.completionHoldMs,
};
}
export async function renderMemoryFlowTui(
input: MemoryFlowReplayInput,
io: KloMemoryFlowTuiIo,
options: RenderMemoryFlowTuiOptions = {},
): Promise<boolean> {
let instance: MemoryFlowInkInstance | null = null;
const paceEvents = options.paceEvents !== false;
const timing = resolveTiming(options);
try {
const onExit = (): void => {
instance?.unmount();
};
instance = (options.renderInk ?? renderInk)(
renderTree(input, io, onExit, { paceEvents, ...timing }),
renderOptions(io),
);
await instance.waitUntilExit();
instance.unmount();
return true;
} catch (error) {
io.stderr.write(`TUI visualization unavailable: ${sanitizeMemoryFlowTuiError(error)}; using text renderer.\n`);
return false;
}
}
export async function startLiveMemoryFlowTui(
input: MemoryFlowReplayInput,
io: KloMemoryFlowTuiIo,
options: StartLiveMemoryFlowTuiOptions = {},
): Promise<MemoryFlowTuiLiveSession | null> {
let instance: MemoryFlowInkInstance | null = null;
let closed = false;
const close = (): void => {
if (closed) {
return;
}
closed = true;
instance?.unmount();
};
try {
instance = (options.renderInk ?? renderInk)(renderTree(input, io, close), renderOptions(io));
return {
update(nextInput: MemoryFlowReplayInput): void {
if (closed) {
return;
}
instance?.rerender(renderTree(nextInput, io, close));
},
close,
isClosed(): boolean {
return closed;
},
};
} catch (error) {
io.stderr.write(`TUI visualization unavailable: ${sanitizeMemoryFlowTuiError(error)}; using text renderer.\n`);
return null;
}
}

View file

@ -0,0 +1,129 @@
import { describe, expect, it } from 'vitest';
import {
KLO_CONTEXT_BUILD_COMMANDS,
KLO_NEXT_STEP_COMMANDS,
formatNextStepLines,
formatSetupNextStepLines,
} from './next-steps.js';
const command = (...parts: string[]) => parts.join(' ');
describe('KLO demo next steps', () => {
it('uses supported context-build commands before agent usage', () => {
expect(KLO_CONTEXT_BUILD_COMMANDS).toEqual([
{
command: 'klo setup context build',
description: 'Build agent-ready context from configured primary and context sources',
},
{
command: 'klo status',
description: 'Check setup and context readiness',
},
{
command: 'klo setup context status',
description: 'Check the setup-managed context build state',
},
]);
});
it('uses supported final public commands', () => {
expect(KLO_NEXT_STEP_COMMANDS).toEqual([
{
command: 'klo agent context --json',
description: 'Verify the project context your agent can read',
},
{
command: 'klo agent tools --json',
description: 'List direct CLI tools available to agents',
},
{
command: 'klo sl list',
description: 'Inspect generated semantic-layer sources',
},
{
command: 'klo wiki list',
description: 'Inspect generated wiki pages',
},
{
command: 'klo serve --mcp stdio --user-id local',
description: 'Optional MCP server route for clients that require MCP',
},
]);
});
it('prefers the direct CLI route before MCP serving', () => {
const commands = KLO_NEXT_STEP_COMMANDS.map((step) => step.command);
expect(commands.indexOf('klo agent context --json')).toBeLessThan(
commands.indexOf('klo serve --mcp stdio --user-id local'),
);
expect(commands.indexOf('klo agent tools --json')).toBeLessThan(
commands.indexOf('klo serve --mcp stdio --user-id local'),
);
});
it('explains what the next-step commands are for', () => {
const rendered = formatNextStepLines().join('\n');
expect(rendered).toContain('KLO context is ready for agents.');
expect(rendered).toContain('Preferred route: CLI + Skills');
expect(rendered).toContain('no MCP server is required');
expect(rendered).toContain('Direct CLI checks:');
expect(rendered).toContain('Optional MCP:');
expect(rendered).not.toContain('Ask your agent to use KLO');
});
it('does not advertise removed Commander migration commands', () => {
const rendered = formatNextStepLines().join('\n');
expect(rendered).toContain('klo agent tools --json');
expect(rendered).toContain('klo agent context --json');
expect(rendered).toContain('klo sl list');
expect(rendered).toContain('klo wiki list');
expect(rendered).toContain('klo serve --mcp stdio --user-id local');
for (const removed of [
command('klo', 'ask'),
command('klo', 'mcp'),
command('klo', 'connect'),
command('klo', 'knowledge'),
command('dev', 'model'),
command('dev', 'knowledge'),
command('klo', 'ingest', 'run'),
command('klo', 'ingest', 'replay'),
]) {
expect(rendered).not.toContain(removed);
}
});
it('keeps setup next steps focused on building context when the build is not ready', () => {
const rendered = formatSetupNextStepLines({
setupReady: true,
hasContextTargets: true,
contextReady: false,
agentIntegrationReady: true,
}).join('\n');
expect(rendered).toContain('Build KLO context next.');
expect(rendered).toContain('primary-source scans and context-source ingests');
expect(rendered).toContain('klo setup context build');
expect(rendered).toContain('klo status');
expect(rendered).toContain('klo setup context status');
expect(rendered).not.toContain('klo agent context --json');
expect(rendered).not.toContain('klo serve --mcp');
});
it('shows agent commands only after setup and context build are ready', () => {
const rendered = formatSetupNextStepLines({
setupReady: true,
hasContextTargets: true,
contextReady: true,
agentIntegrationReady: true,
}).join('\n');
expect(rendered).toContain('KLO context is ready for agents.');
expect(rendered).toContain('klo agent context --json');
expect(rendered).toContain('klo serve --mcp stdio --user-id local');
expect(rendered).not.toContain('Build KLO context next.');
});
});

View file

@ -0,0 +1,104 @@
export const KLO_CONTEXT_BUILD_COMMANDS = [
{
command: 'klo setup context build',
description: 'Build agent-ready context from configured primary and context sources',
},
{
command: 'klo status',
description: 'Check setup and context readiness',
},
{
command: 'klo setup context status',
description: 'Check the setup-managed context build state',
},
] as const;
export const KLO_NEXT_STEP_DIRECT_COMMANDS = [
{
command: 'klo agent context --json',
description: 'Verify the project context your agent can read',
},
{
command: 'klo agent tools --json',
description: 'List direct CLI tools available to agents',
},
{
command: 'klo sl list',
description: 'Inspect generated semantic-layer sources',
},
{
command: 'klo wiki list',
description: 'Inspect generated wiki pages',
},
] as const;
export const KLO_NEXT_STEP_MCP_COMMANDS = [
{
command: 'klo serve --mcp stdio --user-id local',
description: 'Optional MCP server route for clients that require MCP',
},
] as const;
export const KLO_NEXT_STEP_COMMANDS = [...KLO_NEXT_STEP_DIRECT_COMMANDS, ...KLO_NEXT_STEP_MCP_COMMANDS] as const;
export const KLO_NEXT_STEP_COMMAND_WIDTH = Math.max(
...[...KLO_CONTEXT_BUILD_COMMANDS, ...KLO_NEXT_STEP_COMMANDS].map((step) => step.command.length),
);
export interface KloSetupNextStepState {
setupReady: boolean;
hasContextTargets: boolean;
contextReady: boolean;
agentIntegrationReady: boolean;
}
function commandLines(commands: ReadonlyArray<{ command: string; description: string }>, indent: string): string[] {
return commands.map((step) => `${indent}$ ${step.command.padEnd(KLO_NEXT_STEP_COMMAND_WIDTH)} ${step.description}`);
}
export function formatNextStepLines(indent = ' '): string[] {
return [
`${indent}KLO context is ready for agents.`,
`${indent}Preferred route: CLI + Skills; installed rules call \`klo agent ...\` directly, so no MCP server is required.`,
`${indent}Direct CLI checks:`,
...commandLines(KLO_NEXT_STEP_DIRECT_COMMANDS, indent),
`${indent}Optional MCP:`,
...commandLines(KLO_NEXT_STEP_MCP_COMMANDS, indent),
];
}
export function formatSetupNextStepLines(state: KloSetupNextStepState, indent = ' '): string[] {
if (!state.setupReady) {
return [
`${indent}Finish setup first.`,
`${indent}$ ${'klo setup'.padEnd(KLO_NEXT_STEP_COMMAND_WIDTH)} Resume configuration and validation`,
`${indent}$ ${'klo status'.padEnd(KLO_NEXT_STEP_COMMAND_WIDTH)} Check which setup steps still need attention`,
];
}
if (!state.hasContextTargets) {
return [
`${indent}Connect data, then build context.`,
`${indent}$ ${'klo setup'.padEnd(KLO_NEXT_STEP_COMMAND_WIDTH)} Add primary or context sources`,
`${indent}$ ${'klo status'.padEnd(KLO_NEXT_STEP_COMMAND_WIDTH)} Check setup and context readiness`,
];
}
if (!state.contextReady) {
return [
`${indent}Build KLO context next.`,
`${indent}Preferred route: run the CLI build; it covers primary-source scans and context-source ingests.`,
...commandLines(KLO_CONTEXT_BUILD_COMMANDS, indent),
];
}
if (!state.agentIntegrationReady) {
return [
`${indent}KLO context is built. Install agent rules when you want your coding agent to use it.`,
`${indent}$ ${'klo setup --agents'.padEnd(KLO_NEXT_STEP_COMMAND_WIDTH)} Install CLI-based agent rules`,
`${indent}$ ${'klo status'.padEnd(KLO_NEXT_STEP_COMMAND_WIDTH)} Check setup and context readiness`,
];
}
return formatNextStepLines(indent);
}

View file

@ -0,0 +1,172 @@
import { afterEach, describe, expect, it, vi } from 'vitest';
import { runKloCli, type KloCliDeps } from './index.js';
function makeIo() {
let stdout = '';
let stderr = '';
return {
io: {
stdout: {
write: (chunk: string) => {
stdout += chunk;
},
},
stderr: {
write: (chunk: string) => {
stderr += chunk;
},
},
},
stdout: () => stdout,
stderr: () => stderr,
};
}
describe('project directory defaults', () => {
afterEach(() => {
delete process.env.KLO_PROJECT_DIR;
});
it('uses KLO_PROJECT_DIR when Commander-dispatched commands omit --project-dir', async () => {
process.env.KLO_PROJECT_DIR = '/tmp/klo-env-project';
const connection = vi.fn(async () => 0);
const demo = vi.fn(async () => 0);
const doctor = vi.fn(async () => 0);
const ingest = vi.fn(async () => 0);
const publicIngest = vi.fn(async () => 0);
const scan = vi.fn(async () => 0);
const serveStdio = vi.fn(async () => 0);
const setup = vi.fn(async () => 0);
const agent = vi.fn(async () => 0);
const deps: KloCliDeps = { agent, connection, demo, doctor, ingest, publicIngest, scan, serveStdio, setup };
const cases: Array<{
argv: string[];
spy: ReturnType<typeof vi.fn>;
expected: Record<string, unknown>;
runnerType: 'cli' | 'serve';
}> = [
{
argv: ['connection', 'list'],
spy: connection,
expected: { command: 'list', projectDir: '/tmp/klo-env-project' },
runnerType: 'cli',
},
{
argv: ['setup', 'demo', 'scan', '--no-input'],
spy: demo,
expected: { command: 'scan', projectDir: '/tmp/klo-env-project' },
runnerType: 'cli',
},
{
argv: ['dev', 'doctor', '--no-input'],
spy: doctor,
expected: { command: 'project', projectDir: '/tmp/klo-env-project' },
runnerType: 'cli',
},
{
argv: ['ingest', 'status', 'run-1'],
spy: publicIngest,
expected: { command: 'status', projectDir: '/tmp/klo-env-project', runId: 'run-1' },
runnerType: 'cli',
},
{
argv: ['setup', 'status'],
spy: setup,
expected: { command: 'status', projectDir: '/tmp/klo-env-project' },
runnerType: 'cli',
},
{
argv: ['dev', 'scan', 'warehouse'],
spy: scan,
expected: { command: 'run', projectDir: '/tmp/klo-env-project', connectionId: 'warehouse' },
runnerType: 'cli',
},
{
argv: ['serve', '--mcp', 'stdio'],
spy: serveStdio,
expected: { mcp: 'stdio', projectDir: '/tmp/klo-env-project' },
runnerType: 'serve',
},
{
argv: ['agent', 'tools', '--json'],
spy: agent,
expected: { command: 'tools', projectDir: '/tmp/klo-env-project' },
runnerType: 'cli',
},
];
for (const item of cases) {
const testIo = makeIo();
await expect(runKloCli(item.argv, testIo.io, deps)).resolves.toBe(0);
if (item.runnerType === 'serve') {
expect(item.spy).toHaveBeenLastCalledWith(expect.objectContaining(item.expected));
} else {
expect(item.spy).toHaveBeenLastCalledWith(expect.objectContaining(item.expected), testIo.io);
}
expect(testIo.stderr()).toBe('');
}
});
it('lets explicit global --project-dir override KLO_PROJECT_DIR before and after nested commands', async () => {
process.env.KLO_PROJECT_DIR = '/tmp/klo-env-project';
const scan = vi.fn(async () => 0);
const publicIngest = vi.fn(async () => 0);
const scanIo = makeIo();
const ingestIo = makeIo();
await expect(
runKloCli(['--project-dir', '/tmp/klo-explicit-project', 'dev', 'scan', 'warehouse'], scanIo.io, { scan }),
).resolves.toBe(0);
await expect(
runKloCli(['ingest', 'status', 'run-1', '--project-dir=/tmp/klo-explicit-project'], ingestIo.io, {
publicIngest,
}),
).resolves.toBe(0);
expect(scan).toHaveBeenCalledWith(
expect.objectContaining({ command: 'run', projectDir: '/tmp/klo-explicit-project' }),
scanIo.io,
);
expect(publicIngest).toHaveBeenCalledWith(
expect.objectContaining({ command: 'status', projectDir: '/tmp/klo-explicit-project' }),
ingestIo.io,
);
expect(scanIo.stderr()).toBe('');
expect(ingestIo.stderr()).toBe('');
});
it('uses nearest ancestor containing klo.yaml when no explicit or environment project-dir exists', async () => {
const { mkdir, realpath, writeFile } = await import('node:fs/promises');
const { mkdtemp, rm } = await import('node:fs/promises');
const { tmpdir } = await import('node:os');
const { join } = await import('node:path');
const originalCwd = process.cwd();
const root = await mkdtemp(join(tmpdir(), 'klo-cli-nearest-project-'));
const projectDir = join(root, 'warehouse');
const nestedDir = join(projectDir, 'nested', 'deeper');
await mkdir(nestedDir, { recursive: true });
await writeFile(join(projectDir, 'klo.yaml'), 'project: warehouse\n', 'utf-8');
const expectedProjectDir = await realpath(projectDir);
const scan = vi.fn(async () => 0);
const testIo = makeIo();
try {
process.chdir(nestedDir);
await expect(runKloCli(['dev', 'scan', 'warehouse'], testIo.io, { scan })).resolves.toBe(0);
} finally {
process.chdir(originalCwd);
await rm(root, { recursive: true, force: true });
}
expect(scan).toHaveBeenCalledWith(
expect.objectContaining({ command: 'run', projectDir: expectedProjectDir }),
testIo.io,
);
expect(testIo.stderr()).toBe('');
});
});

View file

@ -0,0 +1,5 @@
import { resolve } from 'node:path';
export function resolveProjectDir(projectDir?: string, fallback = '.'): string {
return resolve(projectDir ?? fallback);
}

View file

@ -0,0 +1,70 @@
import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join, resolve } from 'node:path';
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
import { findNearestKloProjectDir, resolveKloProjectDir } from './project-resolver.js';
describe('resolveKloProjectDir', () => {
let tempDir: string;
beforeEach(async () => {
tempDir = await mkdtemp(join(tmpdir(), 'klo-project-resolver-'));
});
afterEach(async () => {
await rm(tempDir, { recursive: true, force: true });
});
it('prefers an explicit project directory', async () => {
const explicit = join(tempDir, 'explicit');
const envProject = join(tempDir, 'env');
await mkdir(explicit, { recursive: true });
await mkdir(envProject, { recursive: true });
expect(
resolveKloProjectDir({
explicitProjectDir: explicit,
env: { KLO_PROJECT_DIR: envProject },
cwd: tempDir,
}),
).toBe(resolve(explicit));
});
it('uses KLO_PROJECT_DIR when no explicit project directory is set', async () => {
const envProject = join(tempDir, 'env-project');
await mkdir(envProject, { recursive: true });
expect(resolveKloProjectDir({ env: { KLO_PROJECT_DIR: envProject }, cwd: tempDir })).toBe(resolve(envProject));
});
it('resolves a relative KLO_PROJECT_DIR value from cwd', () => {
expect(resolveKloProjectDir({ env: { KLO_PROJECT_DIR: 'env-project' }, cwd: tempDir })).toBe(
resolve(tempDir, 'env-project'),
);
});
it('uses the nearest ancestor containing klo.yaml', async () => {
const project = join(tempDir, 'warehouse');
const nested = join(project, 'nested', 'deeper');
await mkdir(nested, { recursive: true });
await writeFile(join(project, 'klo.yaml'), 'project: warehouse\n', 'utf-8');
expect(resolveKloProjectDir({ env: {}, cwd: nested })).toBe(resolve(project));
expect(findNearestKloProjectDir(nested)).toBe(resolve(project));
});
it('falls back to the current directory when no project marker exists', () => {
expect(resolveKloProjectDir({ env: {}, cwd: tempDir })).toBe(resolve(tempDir));
expect(findNearestKloProjectDir(tempDir)).toBeUndefined();
});
it('rejects empty explicit and environment project directory values', () => {
expect(() => resolveKloProjectDir({ explicitProjectDir: ' ', cwd: tempDir })).toThrow(
'--project-dir requires a value',
);
expect(() => resolveKloProjectDir({ env: { KLO_PROJECT_DIR: ' ' }, cwd: tempDir })).toThrow(
'KLO_PROJECT_DIR must not be empty',
);
});
});

View file

@ -0,0 +1,56 @@
import { existsSync } from 'node:fs';
import { dirname, join, resolve } from 'node:path';
export interface KloProjectResolverOptions {
explicitProjectDir?: string;
env?: Partial<Pick<NodeJS.ProcessEnv, 'KLO_PROJECT_DIR'>>;
cwd?: string;
}
function nonEmptyValue(value: string | undefined): string | undefined {
if (value === undefined) {
return undefined;
}
const trimmed = value.trim();
return trimmed.length > 0 ? value : undefined;
}
export function findNearestKloProjectDir(startDir = process.cwd()): string | undefined {
let current = resolve(startDir);
while (true) {
if (existsSync(join(current, 'klo.yaml'))) {
return current;
}
const parent = dirname(current);
if (parent === current) {
return undefined;
}
current = parent;
}
}
export function resolveKloProjectDir(options: KloProjectResolverOptions = {}): string {
const cwd = options.cwd ?? process.cwd();
if (options.explicitProjectDir !== undefined) {
const explicit = nonEmptyValue(options.explicitProjectDir);
if (!explicit) {
throw new Error('--project-dir requires a value');
}
return resolve(cwd, explicit);
}
const rawEnvProjectDir = options.env ? options.env.KLO_PROJECT_DIR : process.env.KLO_PROJECT_DIR;
const envProjectDir = nonEmptyValue(rawEnvProjectDir);
if (rawEnvProjectDir !== undefined && envProjectDir === undefined) {
throw new Error('KLO_PROJECT_DIR must not be empty');
}
if (envProjectDir !== undefined) {
return resolve(cwd, envProjectDir);
}
const resolvedCwd = resolve(cwd);
return findNearestKloProjectDir(resolvedCwd) ?? resolvedCwd;
}

View file

@ -0,0 +1,48 @@
import { describe, expect, it } from 'vitest';
import { withMenuOptionSpacing, withMultiselectNavigation, withTextInputNavigation } from './prompt-navigation.js';
describe('prompt navigation helpers', () => {
it('leaves compact single-line menu prompts unchanged', () => {
expect(withMenuOptionSpacing('What do you want to do?')).toBe('What do you want to do?');
});
it('adds a blank separator between multiline menu copy and the option list', () => {
expect(withMenuOptionSpacing('Which embedding option should KLO use?\n\nKLO uses embeddings for search.')).toBe(
'Which embedding option should KLO use?\n\nKLO uses embeddings for search.\n',
);
});
it('does not duplicate an existing option-list separator', () => {
expect(withMenuOptionSpacing('Question\n\nContext\n')).toBe('Question\n\nContext\n');
});
it('keeps multiselect navigation copy multiline so menu renderers can separate it from options', () => {
expect(withMultiselectNavigation('Which sources?')).toBe(
'Which sources?\nUse Up/Down to move, Space to select or unselect, Enter to confirm, Escape to go back, or Ctrl+C to exit.',
);
});
it('adds a blank separator between text input helper copy and the editable value', () => {
expect(
withTextInputNavigation(
'Name this PostgreSQL connection\nKLO will use this short name in commands and config. You can rename it now.',
),
).toBe(
'Name this PostgreSQL connection\n\nKLO will use this short name in commands and config. You can rename it now.\nPress Escape to go back.\n',
);
});
it('adds a blank separator before compact text input values', () => {
expect(withTextInputNavigation('Project folder path')).toBe('Project folder path\nPress Escape to go back.\n');
});
it('normalizes already hinted text input prompts without duplicating the hint', () => {
expect(
withTextInputNavigation(
'Name this PostgreSQL connection\nKLO will use this short name in commands and config. You can rename it now.\nPress Escape to go back.',
),
).toBe(
'Name this PostgreSQL connection\n\nKLO will use this short name in commands and config. You can rename it now.\nPress Escape to go back.\n',
);
});
});

View file

@ -0,0 +1,45 @@
const MULTISELECT_MENU_NAVIGATION_HINT =
'Use Up/Down to move, Space to select or unselect, Enter to confirm, Escape to go back, or Ctrl+C to exit.';
const TEXT_INPUT_NAVIGATION_HINT = 'Press Escape to go back.';
function removeTrailingBlankLines(message: string): string {
return message.replace(/\n+$/, '');
}
function withTextInputBodySpacing(message: string): string {
const normalized = removeTrailingBlankLines(message);
if (!normalized.includes('\n')) {
return normalized;
}
const [title, ...bodyLines] = normalized.split('\n');
if (bodyLines[0] === '') {
return normalized;
}
return `${title}\n\n${bodyLines.join('\n')}`;
}
export function withMenuOptionSpacing(message: string): string {
if (!message.includes('\n') || message.endsWith('\n')) {
return message;
}
return `${message}\n`;
}
export function withMenuOptionsSpacing<T extends { message: string }>(options: T): T {
return { ...options, message: withMenuOptionSpacing(options.message) };
}
export function withMultiselectNavigation(message: string): string {
if (message.includes(MULTISELECT_MENU_NAVIGATION_HINT)) {
return message;
}
return `${message}\n${MULTISELECT_MENU_NAVIGATION_HINT}`;
}
export function withTextInputNavigation(message: string): string {
const messageWithoutHint = removeTrailingBlankLines(message)
.split('\n')
.filter((line) => line !== TEXT_INPUT_NAVIGATION_HINT)
.join('\n');
return `${withTextInputBodySpacing(messageWithoutHint)}\n${TEXT_INPUT_NAVIGATION_HINT}\n`;
}

View file

@ -0,0 +1,292 @@
import { buildDefaultKloProjectConfig, type KloProjectConfig } from '@klo/context/project';
import { describe, expect, it, vi } from 'vitest';
import { buildPublicIngestPlan, type KloPublicIngestProject, runKloPublicIngest } from './public-ingest.js';
function makeIo(options: { isTTY?: boolean } = {}) {
let stdout = '';
let stderr = '';
return {
io: {
stdout: {
isTTY: options.isTTY,
write: (chunk: string) => {
stdout += chunk;
},
},
stderr: {
write: (chunk: string) => {
stderr += chunk;
},
},
},
stdout: () => stdout,
stderr: () => stderr,
};
}
function projectWithConnections(connections: KloProjectConfig['connections']): KloPublicIngestProject {
return {
projectDir: '/tmp/project',
config: {
...buildDefaultKloProjectConfig('warehouse'),
connections,
},
};
}
describe('buildPublicIngestPlan', () => {
it('plans warehouse connections as scan targets and source connections as source ingest targets', () => {
const project = projectWithConnections({
warehouse: { driver: 'postgres' },
prod_metabase: { driver: 'metabase' },
docs: { driver: 'notion' },
});
expect(buildPublicIngestPlan(project, { projectDir: '/tmp/project', all: true })).toEqual({
projectDir: '/tmp/project',
targets: [
{
connectionId: 'warehouse',
driver: 'postgres',
operation: 'scan',
debugCommand: 'klo scan warehouse --debug',
steps: ['scan'],
},
{
connectionId: 'docs',
driver: 'notion',
operation: 'source-ingest',
adapter: 'notion',
debugCommand: 'klo dev ingest run --connection-id docs --adapter notion --debug',
steps: ['source-ingest', 'memory-update'],
},
{
connectionId: 'prod_metabase',
driver: 'metabase',
operation: 'source-ingest',
adapter: 'metabase',
debugCommand: 'klo dev ingest run --connection-id prod_metabase --adapter metabase --debug',
steps: ['source-ingest', 'memory-update'],
},
],
});
});
it('rejects bare non-interactive ingest until the interactive confirmation slice exists', () => {
const project = projectWithConnections({ warehouse: { driver: 'postgres' } });
expect(() => buildPublicIngestPlan(project, { projectDir: '/tmp/project', all: false })).toThrow(
'klo ingest requires <connectionId> or --all in this release',
);
});
it('does not plan PostHog connections as CLI ingest targets', () => {
const project = projectWithConnections({ product: { driver: 'posthog' } });
expect(() =>
buildPublicIngestPlan(project, { projectDir: '/tmp/project', targetConnectionId: 'product', all: false }),
).toThrow('Connection "product" uses unsupported public ingest driver "posthog"');
});
});
describe('runKloPublicIngest', () => {
it('runs all independent targets and reports partial failures', async () => {
const io = makeIo();
const project = projectWithConnections({
warehouse: { driver: 'postgres' },
prod_metabase: { driver: 'metabase' },
});
const runScan = vi.fn(async () => 1);
const runIngest = vi.fn(async () => 0);
await expect(
runKloPublicIngest(
{ command: 'run', projectDir: '/tmp/project', all: true, json: false, inputMode: 'disabled' },
io.io,
{
loadProject: vi.fn(async () => project),
runScan,
runIngest,
},
),
).resolves.toBe(1);
expect(runIngest).toHaveBeenCalledWith(
{
command: 'run',
projectDir: '/tmp/project',
connectionId: 'prod_metabase',
adapter: 'metabase',
outputMode: 'plain',
inputMode: 'disabled',
},
expect.anything(),
);
expect(runScan).toHaveBeenCalledWith(
{
command: 'run',
projectDir: '/tmp/project',
connectionId: 'warehouse',
mode: 'structural',
detectRelationships: false,
dryRun: false,
},
expect.anything(),
);
expect(io.stdout()).toContain('Ingest finished with partial failures');
expect(io.stdout()).toContain('warehouse failed at scan.');
expect(io.stdout()).toContain('Debug: klo scan warehouse --debug');
});
it('can request enriched relationship scans for setup-managed context builds', async () => {
const io = makeIo();
const project = projectWithConnections({ warehouse: { driver: 'postgres' } });
const runScan = vi.fn(async () => 0);
await expect(
runKloPublicIngest(
{
command: 'run',
projectDir: '/tmp/project',
all: true,
json: false,
inputMode: 'disabled',
scanMode: 'enriched',
detectRelationships: true,
},
io.io,
{
loadProject: vi.fn(async () => project),
runScan,
},
),
).resolves.toBe(0);
expect(runScan).toHaveBeenCalledWith(
{
command: 'run',
projectDir: '/tmp/project',
connectionId: 'warehouse',
mode: 'enriched',
detectRelationships: true,
dryRun: false,
},
io.io,
);
});
it('prints stable JSON results', async () => {
const io = makeIo();
const project = projectWithConnections({ warehouse: { driver: 'postgres' } });
await expect(
runKloPublicIngest(
{
command: 'run',
projectDir: '/tmp/project',
targetConnectionId: 'warehouse',
all: false,
json: true,
inputMode: 'disabled',
},
io.io,
{
loadProject: vi.fn(async () => project),
runScan: vi.fn(async () => 0),
},
),
).resolves.toBe(0);
expect(JSON.parse(io.stdout())).toMatchObject({
plan: { projectDir: '/tmp/project' },
results: [{ connectionId: 'warehouse', driver: 'postgres' }],
});
});
it('passes dbt source_dir from connection config to runKloIngest', async () => {
const runIngest = vi.fn(async () => 0);
const io = makeIo();
await expect(
runKloPublicIngest(
{
command: 'run',
projectDir: '/tmp/klo',
targetConnectionId: 'analytics_dbt',
all: false,
json: false,
inputMode: 'disabled',
},
io.io,
{
loadProject: async () =>
({
projectDir: '/tmp/klo',
config: {
connections: {
analytics_dbt: {
driver: 'dbt',
source_dir: '/repo/dbt',
},
},
},
}) as never,
runIngest,
},
),
).resolves.toBe(0);
expect(runIngest).toHaveBeenCalledWith(
expect.objectContaining({
command: 'run',
connectionId: 'analytics_dbt',
adapter: 'dbt',
sourceDir: '/repo/dbt',
}),
io.io,
);
});
it('routes public status and watch to the ingest status renderer', async () => {
const runIngest = vi.fn(async () => 0);
const statusIo = makeIo();
const watchIo = makeIo();
await expect(
runKloPublicIngest(
{ command: 'status', projectDir: '/tmp/klo', json: false, inputMode: 'disabled' },
statusIo.io,
{ runIngest },
),
).resolves.toBe(0);
await expect(
runKloPublicIngest(
{ command: 'watch', projectDir: '/tmp/klo', runId: 'run-1', json: false, inputMode: 'auto' },
watchIo.io,
{ runIngest },
),
).resolves.toBe(0);
expect(runIngest).toHaveBeenNthCalledWith(
1,
{
command: 'status',
projectDir: '/tmp/klo',
outputMode: 'plain',
inputMode: 'disabled',
},
statusIo.io,
);
expect(runIngest).toHaveBeenNthCalledWith(
2,
{
command: 'watch',
projectDir: '/tmp/klo',
runId: 'run-1',
outputMode: 'viz',
inputMode: 'auto',
},
watchIo.io,
);
});
});

View file

@ -0,0 +1,315 @@
import { type KloLocalProject, type KloProjectConnectionConfig, loadKloProject } from '@klo/context/project';
import type { KloCliIo } from './index.js';
import type { KloIngestArgs } from './ingest.js';
import type { KloScanArgs } from './scan.js';
import { profileMark } from './startup-profile.js';
profileMark('module:public-ingest');
export type KloPublicIngestStepName = 'scan' | 'source-ingest' | 'enrich' | 'memory-update';
export type KloPublicIngestStepStatus = 'done' | 'skipped' | 'failed' | 'not-run';
export type KloPublicIngestInputMode = 'auto' | 'disabled';
export type KloPublicIngestArgs =
| {
command: 'run';
projectDir: string;
targetConnectionId?: string;
all: boolean;
json: boolean;
inputMode: KloPublicIngestInputMode;
scanMode?: Extract<KloScanArgs, { command: 'run' }>['mode'];
detectRelationships?: boolean;
}
| {
command: 'status' | 'watch';
projectDir: string;
runId?: string;
json: boolean;
inputMode: KloPublicIngestInputMode;
};
export interface KloPublicIngestPlanTarget {
connectionId: string;
driver: string;
operation: 'scan' | 'source-ingest';
adapter?: string;
sourceDir?: string;
debugCommand: string;
steps: KloPublicIngestStepName[];
}
export interface KloPublicIngestPlan {
projectDir: string;
targets: KloPublicIngestPlanTarget[];
}
export interface KloPublicIngestTargetResult {
connectionId: string;
driver: string;
steps: Array<{
operation: KloPublicIngestStepName;
status: KloPublicIngestStepStatus;
detail?: string;
debugCommand?: string;
}>;
}
export type KloPublicIngestProject = Pick<KloLocalProject, 'projectDir' | 'config'>;
export interface KloPublicIngestDeps {
loadProject?: (options: Parameters<typeof loadKloProject>[0]) => Promise<KloPublicIngestProject>;
runScan?: (args: KloScanArgs, io: KloCliIo) => Promise<number>;
runIngest?: (args: KloIngestArgs, io: KloCliIo) => Promise<number>;
}
const sourceAdapterByDriver = new Map<string, string>([
['metabase', 'metabase'],
['local_metabase', 'metabase'],
['looker', 'looker'],
['local_looker', 'looker'],
['notion', 'notion'],
['metricflow', 'metricflow'],
['dbt', 'dbt'],
['lookml', 'lookml'],
]);
const warehouseDrivers = new Set([
'sqlite',
'postgres',
'postgresql',
'mysql',
'clickhouse',
'sqlserver',
'bigquery',
'snowflake',
]);
function normalizedDriver(connection: KloProjectConnectionConfig): string {
return String(connection.driver ?? '')
.trim()
.toLowerCase();
}
function sourceDirForConnection(connection: KloProjectConnectionConfig): string | undefined {
const value = connection.source_dir ?? connection.sourceDir;
return typeof value === 'string' && value.trim().length > 0 ? value.trim() : undefined;
}
function targetForConnection(connectionId: string, connection: KloProjectConnectionConfig): KloPublicIngestPlanTarget {
const driver = normalizedDriver(connection);
const adapter = sourceAdapterByDriver.get(driver);
const sourceDir = sourceDirForConnection(connection);
if (adapter) {
return {
connectionId,
driver,
operation: 'source-ingest',
adapter,
...(sourceDir ? { sourceDir } : {}),
debugCommand: `klo dev ingest run --connection-id ${connectionId} --adapter ${adapter} --debug`,
steps: ['source-ingest', 'memory-update'],
};
}
if (warehouseDrivers.has(driver)) {
return {
connectionId,
driver,
operation: 'scan',
debugCommand: `klo scan ${connectionId} --debug`,
steps: ['scan'],
};
}
throw new Error(`Connection "${connectionId}" uses unsupported public ingest driver "${driver || 'unknown'}"`);
}
export function buildPublicIngestPlan(
project: KloPublicIngestProject,
args: { projectDir: string; targetConnectionId?: string; all: boolean },
): KloPublicIngestPlan {
if (!args.all && !args.targetConnectionId) {
throw new Error('klo ingest requires <connectionId> or --all in this release');
}
const entries = Object.entries(project.config.connections).sort(([a], [b]) => a.localeCompare(b));
const selected = args.all ? entries : entries.filter(([connectionId]) => connectionId === args.targetConnectionId);
if (!args.all && selected.length === 0) {
throw new Error(`Connection "${args.targetConnectionId}" is not configured in klo.yaml`);
}
if (selected.length === 0) {
throw new Error('No configured connections are eligible for ingest');
}
const targets = selected.map(([connectionId, connection]) => targetForConnection(connectionId, connection));
return {
projectDir: args.projectDir,
targets: [...targets.filter((t) => t.operation === 'scan'), ...targets.filter((t) => t.operation === 'source-ingest')],
};
}
function defaultSteps(target: KloPublicIngestPlanTarget): KloPublicIngestTargetResult['steps'] {
return [
{
operation: 'scan',
status: target.steps.includes('scan') ? 'not-run' : 'skipped',
...(target.operation === 'scan' ? { debugCommand: target.debugCommand } : {}),
},
{
operation: 'source-ingest',
status: target.steps.includes('source-ingest') ? 'not-run' : 'skipped',
...(target.operation === 'source-ingest' ? { debugCommand: target.debugCommand } : {}),
},
{ operation: 'enrich', status: 'skipped' },
{
operation: 'memory-update',
status: target.steps.includes('memory-update') ? 'not-run' : 'skipped',
...(target.operation === 'source-ingest' ? { debugCommand: target.debugCommand } : {}),
},
];
}
function markTargetResult(target: KloPublicIngestPlanTarget, status: 'done' | 'failed'): KloPublicIngestTargetResult {
const failedOperation = target.operation === 'scan' ? 'scan' : 'source-ingest';
return {
connectionId: target.connectionId,
driver: target.driver,
steps: defaultSteps(target).map((step) => {
if (!target.steps.includes(step.operation)) {
return step;
}
if (status === 'done') {
return { ...step, status: 'done' };
}
if (step.operation === failedOperation) {
return { ...step, status: 'failed', detail: `${target.connectionId} failed at ${failedOperation}.` };
}
return { ...step, status: 'not-run' };
}),
};
}
function resultFailed(result: KloPublicIngestTargetResult): boolean {
return result.steps.some((step) => step.status === 'failed');
}
function stepStatus(result: KloPublicIngestTargetResult, operation: KloPublicIngestStepName): string {
return result.steps.find((step) => step.operation === operation)?.status ?? 'not-run';
}
function renderPlainResults(results: KloPublicIngestTargetResult[], io: KloCliIo): void {
const failures = results.filter(resultFailed);
io.stdout.write(failures.length > 0 ? 'Ingest finished with partial failures\n' : 'Ingest finished\n');
io.stdout.write('\n');
io.stdout.write('Source Scan Source ingest Enrich Memory update\n');
for (const result of results) {
io.stdout.write(
`${result.connectionId.padEnd(14)} ${stepStatus(result, 'scan').padEnd(9)} ${stepStatus(
result,
'source-ingest',
).padEnd(14)} ${stepStatus(result, 'enrich').padEnd(8)} ${stepStatus(result, 'memory-update')}\n`,
);
}
if (failures.length === 0) {
return;
}
io.stdout.write('\nFailed sources:\n');
for (const result of failures) {
const failedStep = result.steps.find((step) => step.status === 'failed');
if (!failedStep) {
continue;
}
io.stdout.write(` ${failedStep.detail ?? `${result.connectionId} failed.`}\n`);
if (failedStep.debugCommand) {
io.stdout.write(` Debug: ${failedStep.debugCommand}\n`);
}
}
}
function hasInteractiveInput(io: KloCliIo): boolean {
const stdin = (io as { stdin?: { isTTY?: boolean; setRawMode?: (value: boolean) => void } }).stdin;
return stdin?.isTTY === true && typeof stdin.setRawMode === 'function';
}
function sourceIngestOutputMode(args: Extract<KloPublicIngestArgs, { command: 'run' }>, io: KloCliIo): 'plain' | 'viz' {
return args.inputMode === 'auto' && io.stdout.isTTY === true && hasInteractiveInput(io) ? 'viz' : 'plain';
}
export async function executePublicIngestTarget(
target: KloPublicIngestPlanTarget,
args: Extract<KloPublicIngestArgs, { command: 'run' }>,
io: KloCliIo,
deps: KloPublicIngestDeps,
): Promise<KloPublicIngestTargetResult> {
if (target.operation === 'scan') {
const { runKloScan } = await import('./scan.js');
const exitCode = await (deps.runScan ?? runKloScan)(
{
command: 'run',
projectDir: args.projectDir,
connectionId: target.connectionId,
mode: args.scanMode ?? 'structural',
detectRelationships: args.detectRelationships ?? false,
dryRun: false,
},
io,
);
return markTargetResult(target, exitCode === 0 ? 'done' : 'failed');
}
const { runKloIngest } = await import('./ingest.js');
const exitCode = await (deps.runIngest ?? runKloIngest)(
{
command: 'run',
projectDir: args.projectDir,
connectionId: target.connectionId,
adapter: target.adapter ?? target.driver,
...(target.sourceDir ? { sourceDir: target.sourceDir } : {}),
outputMode: sourceIngestOutputMode(args, io),
inputMode: args.inputMode,
},
io,
);
return markTargetResult(target, exitCode === 0 ? 'done' : 'failed');
}
export async function runKloPublicIngest(
args: KloPublicIngestArgs,
io: KloCliIo,
deps: KloPublicIngestDeps = {},
): Promise<number> {
if (args.command !== 'run') {
const { runKloIngest } = await import('./ingest.js');
return await (deps.runIngest ?? runKloIngest)(
{
command: args.command,
projectDir: args.projectDir,
...(args.runId ? { runId: args.runId } : {}),
outputMode: args.json ? 'json' : args.command === 'watch' ? 'viz' : 'plain',
inputMode: args.inputMode,
},
io,
);
}
const loadProject = deps.loadProject ?? loadKloProject;
const project = await loadProject({ projectDir: args.projectDir });
const plan = buildPublicIngestPlan(project, args);
const results: KloPublicIngestTargetResult[] = [];
for (const target of plan.targets) {
results.push(await executePublicIngestTarget(target, args, io, deps));
}
if (args.json) {
io.stdout.write(`${JSON.stringify({ plan, results }, null, 2)}\n`);
} else {
renderPlainResults(results, io);
}
return results.some(resultFailed) ? 1 : 0;
}

Some files were not shown because too many files have changed in this diff Show more