mirror of
https://github.com/Kaelio/ktx.git
synced 2026-07-19 11:41:02 +02:00
chore(workspace): gate dead-code with knip production mode (#196)
* refactor(workspace): relocate @ktx/llm source into packages/cli/src/llm * refactor(workspace): rewrite @ktx/llm imports to relative paths * refactor(workspace): fold internal packages into cli * chore(workspace): gate dead-code with knip production mode Turn on production-mode knip plus an autofix run in pre-commit and the `pnpm dead-code` script, document the `/** @internal */` convention for test-only exports in AGENTS.md, annotate test-only exports across the CLI with that JSDoc, and drop dead exports/wrappers the new gate surfaced (e.g. `cli-project.ts`, `lookerRuntimeSourceToFileAdapterSource`, `createLocalScanEnrichmentProvidersFromConfig`, `PGLITE_OWNER_PROCESS_BACKEND_CAPABILITIES`, stale type re-exports). Replace the loose `ignoreIssues` allowlist in `knip.json` with explicit production entries so cross-package barrel leaks are caught. * refactor(cli): delete internal barrel index.ts files The 34 `index.ts` re-export barrels inside `packages/cli/src/` were holdovers from the pre-fold multi-workspace structure. Post-fold-in they served no production purpose: external consumers go through the single package main entry, and in-repo callers mostly imported through them only because the path was short. Internally, knip flagged most barrel re-exports as production-dead (only reached via tests). This change: - Deletes every internal barrel except `packages/cli/src/index.ts` (the published package entry). - Rewrites ~270 source/test files to import each name directly from the file that defines it. - Moves `tools/warehouse-verification/index.ts` to `create-warehouse-verification-tools.ts` (the function it defined locally) and updates its single consumer. - Renames `search/backend-conformance.ts` → `.test-utils.ts` to match the existing test-helper file convention. - Deletes 13 dead test-only chains (dbt-descriptions/*, live-database/extracted-schema, live-database/structural-sync, relationship-* feedback/review chain) plus their tests and a cascading orphan integration test. - Updates test mocks that pointed at deleted barrel paths (notion-client, connector barrels in scan/local-scan-connectors tests) to mock the source files instead. - Points the maintainer benchmark script (`scripts/relationship-benchmark-report.mjs`) at source files instead of `dist/context/scan/index.js`. - Drops the barrel `!` entries from `knip.json`; adds explicit production entries only for the benchmark code reached via dist by the maintainer script. Net: 413 files changed, ~1.2k insertions, ~9.4k deletions. `pnpm run dead-code` (Biome + knip default + knip production) and `pnpm run type-check` are clean; 2277 tests pass. * refactor(workspace): rename @ktx/cli to @kaelio/ktx and pack it directly Promote the CLI workspace package to the public name `@kaelio/ktx` and drop the separate `scripts/build-public-npm-package.mjs` wrapper. The CLI package is now publishable in place (`publishConfig.access: public`, `provenance: true`), so artifact packing uses `pnpm pack` against `packages/cli/` instead of assembling a parallel package tree. Updates all workspace filter invocations, docs, tests, and release readiness checks to reference the new package name, and folds the tarball-name helper into `scripts/public-npm-release-metadata.mjs`. * docs: align "agent clients" and "data agents" terminology Replace "client agents" with "agent clients" and "database agents" with "data agents" across AGENTS.md, README.md, the docs-site copy, and the matching setup-agents test description, matching the canonical vocabulary in docs/terminology.md. Also moves packages/cli/tsconfig.json's tsBuildInfoFile from node_modules/.cache/ to dist/.tsbuildinfo so incremental builds survive node_modules reinstalls. * refactor(release): single source of truth for package version Make packages/cli/package.json the single source of truth for the @kaelio/ktx version. publicNpmPackageVersion() now reads it directly, so artifact filenames, release-readiness checks, and the Python wheel version all derive from one field. The duplicate release-policy.json.publicNpmPackageVersion is removed. Previously the two fields could drift: tarballs were named kaelio-ktx-0.4.1.tgz while internally containing @kaelio/ktx@0.0.0-private. - update-public-release-version.mjs rewrites both Python pyproject.toml files (ktx-daemon, ktx-sl) alongside the npm package.jsons, normalizing the version for PEP 440 (e.g. 0.1.0-rc.2 -> 0.1.0rc2). - semantic-release-config.cjs adds the two pyproject.toml files to @semantic-release/git assets so the release commit back to main carries every version source in lockstep. - The six "?? '0.0.0-private'" fallback literals across the CLI are replaced with "?? getKtxCliPackageInfo().version", and createDefaultKtxMcpServer makes its version arg required. - docs/release.md describes the actual commit-back model: the dev tree always reflects the most recent release; no sentinel pin to maintain. Verified: pnpm run artifacts:build now produces kaelio-ktx-0.4.1.tgz and kaelio_ktx-0.4.1-py3-none-any.whl with @kaelio/ktx@0.4.1 inside. Full type-check, dead-code, and 2287 vitests + 173 script tests pass. * refactor(cli): inject embedding provider resolution and detect sentence-transformers runtime Make resolveProjectEmbeddingProvider and runtimeIo injectable in ingest and scan command entrypoints so tests can stub them, and teach resolvePublicIngestRuntimeRequirements to flag the local-embeddings runtime feature when ktx.yaml selects sentence-transformers. * chore(cli): mark buildLocalStatsStatus and LocalStatsStatus as @internal Both symbols are consumed only by status-project.test.ts. Annotating with /** @internal */ keeps knip's production-mode check clean without changing runtime behavior. * fix(cli): use real package metadata in print-command-tree The stubbed package name embedded a forbidden product identifier that tripped the boundary check in CI. Read the metadata from package.json instead — keeps the rendered tree unchanged and removes a duplicate source of truth. * feat(cli): show embedding coverage in `ktx status`, drop duplicate disk counts Inline `(N embedded)` next to the Wiki scope counts and Semantic-layer source counts, computed with `SUM(embedding_json IS NOT NULL)` over `knowledge_pages` and `local_sl_sources`. Rename the "Knowledge" label to "Wiki" (canonical per `docs/terminology.md`) and rename the matching `localStats.knowledgePages` field to `localStats.wikiPages`. Drop `wiki=N md` and `semantic-layer=N yaml` from the Disk row — those duplicated the per-surface rows above. Disk now reports only actual byte usage (db, cache, raw-sources). The unused `wikiGlobalMarkdownCount` / `semanticLayerYamlCount` fields, the `isMarkdownEntry` / `isYamlEntry` helpers, and the `filter` arg on `summarizeDir` are removed.
This commit is contained in:
parent
a1cfb03d73
commit
2366b00301
1002 changed files with 2286 additions and 12051 deletions
|
|
@ -1,8 +1,7 @@
|
|||
{
|
||||
"name": "@ktx/cli",
|
||||
"name": "@kaelio/ktx",
|
||||
"version": "0.4.1",
|
||||
"description": "CLI wrapper for ktx context packages",
|
||||
"private": true,
|
||||
"description": "Standalone ktx context layer for data agents",
|
||||
"type": "module",
|
||||
"engines": {
|
||||
"node": ">=22.0.0"
|
||||
|
|
@ -24,41 +23,68 @@
|
|||
"dist",
|
||||
"assets"
|
||||
],
|
||||
"publishConfig": {
|
||||
"access": "public",
|
||||
"provenance": true
|
||||
},
|
||||
"scripts": {
|
||||
"assets:demo": "node scripts/build-demo-assets.mjs",
|
||||
"build": "tsc -p tsconfig.json && node scripts/copy-runtime-assets.mjs && node ../../scripts/prepare-cli-bin.mjs",
|
||||
"clean": "node -e \"fs.rmSync('dist', { recursive: true, force: true }); fs.rmSync('node_modules/.cache/tsc.tsbuildinfo', { force: true })\"",
|
||||
"clean": "node -e \"fs.rmSync('dist', { recursive: true, force: true })\"",
|
||||
"docs:commands": "pnpm run build && node dist/print-command-tree.js",
|
||||
"smoke": "vitest run src/standalone-smoke.test.ts src/example-smoke.test.ts --testTimeout 30000",
|
||||
"test": "vitest run --exclude src/standalone-smoke.test.ts --exclude src/example-smoke.test.ts --exclude src/setup-databases.test.ts --exclude src/scan.test.ts --exclude src/commands/connection-metabase-setup.test.ts --exclude src/setup-models.test.ts --exclude src/setup-sources.test.ts --exclude src/setup.test.ts --exclude src/connection.test.ts --exclude src/setup-embeddings.test.ts --exclude src/ingest.test.ts --exclude src/commands/connection-mapping.test.ts --exclude src/ingest-viz.test.ts --exclude src/demo.test.ts --exclude src/setup-project.test.ts --exclude src/sl.test.ts --exclude src/local-scan-connectors.test.ts --exclude src/commands/connection-notion.test.ts",
|
||||
"test:slow": "vitest run src/setup-databases.test.ts src/scan.test.ts src/commands/connection-metabase-setup.test.ts src/setup-models.test.ts src/setup-sources.test.ts src/setup.test.ts src/connection.test.ts src/setup-embeddings.test.ts src/ingest.test.ts src/commands/connection-mapping.test.ts src/ingest-viz.test.ts src/demo.test.ts src/setup-project.test.ts src/sl.test.ts src/local-scan-connectors.test.ts src/commands/connection-notion.test.ts --testTimeout 30000",
|
||||
"type-check": "tsc -p tsconfig.json --noEmit"
|
||||
"test": "vitest run --exclude src/standalone-smoke.test.ts --exclude src/example-smoke.test.ts --exclude src/setup-databases.test.ts --exclude src/scan.test.ts --exclude src/commands/connection-metabase-setup.test.ts --exclude src/setup-models.test.ts --exclude src/setup-sources.test.ts --exclude src/setup.test.ts --exclude src/connection.test.ts --exclude src/setup-embeddings.test.ts --exclude src/ingest.test.ts --exclude src/commands/connection-mapping.test.ts --exclude src/ingest-viz.test.ts --exclude src/demo.test.ts --exclude src/setup-project.test.ts --exclude src/sl.test.ts --exclude src/local-scan-connectors.test.ts --exclude src/commands/connection-notion.test.ts --exclude src/context/scan/local-scan.test.ts --exclude src/context/mcp/local-project-ports.test.ts --exclude src/context/ingest/local-stage-ingest.test.ts --exclude src/context/sl/pglite-sl-search-prototype.test.ts --exclude src/context/core/git.service.test.ts --exclude src/context/ingest/local-adapters.test.ts --exclude src/context/ingest/local-bundle-ingest.test.ts --exclude src/context/ingest/local-metabase-ingest.test.ts --exclude src/context/sl/local-sl.test.ts --exclude src/context/search/pglite-owner-process.test.ts --exclude src/context/scan/local-enrichment-artifacts.test.ts --exclude src/context/search/pglite-spike.test.ts --exclude src/context/wiki/local-knowledge.test.ts --exclude src/context/sl/local-query.test.ts --exclude src/context/scan/relationship-review-decisions.test.ts --exclude src/context/scan/relationship-profiling.test.ts",
|
||||
"test:slow": "vitest run src/setup-databases.test.ts src/scan.test.ts src/commands/connection-metabase-setup.test.ts src/setup-models.test.ts src/setup-sources.test.ts src/setup.test.ts src/connection.test.ts src/setup-embeddings.test.ts src/ingest.test.ts src/commands/connection-mapping.test.ts src/ingest-viz.test.ts src/demo.test.ts src/setup-project.test.ts src/sl.test.ts src/local-scan-connectors.test.ts src/commands/connection-notion.test.ts src/context/scan/local-scan.test.ts src/context/mcp/local-project-ports.test.ts src/context/ingest/local-stage-ingest.test.ts src/context/sl/pglite-sl-search-prototype.test.ts src/context/core/git.service.test.ts src/context/ingest/local-adapters.test.ts src/context/ingest/local-bundle-ingest.test.ts src/context/ingest/local-metabase-ingest.test.ts src/context/sl/local-sl.test.ts src/context/search/pglite-owner-process.test.ts src/context/scan/local-enrichment-artifacts.test.ts src/context/search/pglite-spike.test.ts src/context/wiki/local-knowledge.test.ts src/context/sl/local-query.test.ts src/context/scan/relationship-review-decisions.test.ts src/context/scan/relationship-profiling.test.ts --testTimeout 30000",
|
||||
"type-check": "tsc -p tsconfig.json --noEmit",
|
||||
"relationships:benchmarks": "pnpm --silent run build && node ../../scripts/relationship-benchmark-report.mjs",
|
||||
"relationships:benchmarks:test": "KTX_RUN_RELATIONSHIP_BENCHMARKS=1 vitest run src/context/scan/relationship-benchmarks.test.ts",
|
||||
"search:pglite-spike": "node ../../scripts/pglite-hybrid-search-spike.mjs",
|
||||
"search:pglite-owner-prototype": "node ../../scripts/pglite-owner-process-prototype.mjs",
|
||||
"search:pglite-sl-prototype": "node ../../scripts/pglite-sl-search-prototype.mjs"
|
||||
},
|
||||
"dependencies": {
|
||||
"@ai-sdk/anthropic": "3.0.77",
|
||||
"@ai-sdk/devtools": "0.0.17",
|
||||
"@ai-sdk/google-vertex": "^4.0.128",
|
||||
"@anthropic-ai/claude-agent-sdk": "0.3.142",
|
||||
"@clack/prompts": "1.4.0",
|
||||
"@clickhouse/client": "^1.18.4",
|
||||
"@commander-js/extra-typings": "14.0.0",
|
||||
"@ktx/connector-bigquery": "workspace:*",
|
||||
"@ktx/connector-clickhouse": "workspace:*",
|
||||
"@ktx/connector-mysql": "workspace:*",
|
||||
"@ktx/connector-postgres": "workspace:*",
|
||||
"@ktx/connector-snowflake": "workspace:*",
|
||||
"@ktx/connector-sqlite": "workspace:*",
|
||||
"@ktx/connector-sqlserver": "workspace:*",
|
||||
"@ktx/context": "workspace:*",
|
||||
"@ktx/llm": "workspace:*",
|
||||
"@google-cloud/bigquery": "^8.3.1",
|
||||
"@looker/sdk": "^26.8.0",
|
||||
"@looker/sdk-node": "^26.8.0",
|
||||
"@looker/sdk-rtl": "^21.6.5",
|
||||
"@modelcontextprotocol/sdk": "^1.29.0",
|
||||
"@notionhq/client": "^5.21.0",
|
||||
"ai": "^6.0.180",
|
||||
"better-sqlite3": "^12.10.0",
|
||||
"commander": "14.0.3",
|
||||
"fflate": "^0.8.2",
|
||||
"handlebars": "^4.7.9",
|
||||
"ink": "^7.0.2",
|
||||
"lookml-parser": "7.1.0",
|
||||
"minimatch": "^10.2.5",
|
||||
"mssql": "^12.5.2",
|
||||
"mysql2": "^3.22.3",
|
||||
"openai": "^6.37.0",
|
||||
"p-limit": "^7.3.0",
|
||||
"pg": "^8.20.0",
|
||||
"react": "^19.2.6",
|
||||
"simple-git": "3.36.0",
|
||||
"snowflake-sdk": "^2.4.1",
|
||||
"yaml": "^2.9.0",
|
||||
"zod": "^4.4.3"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@electric-sql/pglite": "^0.4.5",
|
||||
"@electric-sql/pglite-socket": "^0.1.5",
|
||||
"@types/better-sqlite3": "^7.6.13",
|
||||
"@types/mssql": "^12.3.0",
|
||||
"@types/node": "^25.7.0",
|
||||
"@types/pg": "^8.20.0",
|
||||
"@types/react": "^19.2.14",
|
||||
"@vitest/coverage-v8": "^4.1.6",
|
||||
"better-sqlite3": "^12.10.0",
|
||||
"ajv": "8.20.0",
|
||||
"ink-testing-library": "^4.0.0",
|
||||
"typescript": "^6.0.3",
|
||||
"vitest": "^4.1.6"
|
||||
|
|
@ -66,11 +92,11 @@
|
|||
"license": "Apache-2.0",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "git+https://github.com/kaelio/ktx.git",
|
||||
"url": "https://github.com/Kaelio/ktx",
|
||||
"directory": "packages/cli"
|
||||
},
|
||||
"bugs": {
|
||||
"url": "https://github.com/kaelio/ktx/issues"
|
||||
"url": "https://github.com/Kaelio/ktx/issues"
|
||||
},
|
||||
"homepage": "https://github.com/kaelio/ktx#readme"
|
||||
"homepage": "https://github.com/Kaelio/ktx#readme"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,9 +3,14 @@ import { dirname, join } from 'node:path';
|
|||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const packageRoot = fileURLToPath(new URL('..', import.meta.url));
|
||||
const promptsSource = join(packageRoot, 'src', 'prompts');
|
||||
const promptsTarget = join(packageRoot, 'dist', 'prompts');
|
||||
const skillsSource = join(packageRoot, 'src', 'skills');
|
||||
const skillsTarget = join(packageRoot, 'dist', 'skills');
|
||||
|
||||
await rm(promptsTarget, { recursive: true, force: true });
|
||||
await rm(skillsTarget, { recursive: true, force: true });
|
||||
await mkdir(dirname(promptsTarget), { recursive: true });
|
||||
await mkdir(dirname(skillsTarget), { recursive: true });
|
||||
await cp(promptsSource, promptsTarget, { recursive: true });
|
||||
await cp(skillsSource, skillsTarget, { recursive: true });
|
||||
|
|
|
|||
|
|
@ -1,11 +1,11 @@
|
|||
import { createRequire } from 'node:module';
|
||||
|
||||
import type { ReindexSummary } from '@ktx/context/index-sync';
|
||||
import type { ReindexSummary } from './context/index-sync/types.js';
|
||||
import { describe, expect, it, vi } from 'vitest';
|
||||
import { renderReindexJson, renderReindexPlain, reindexHasErrors } from './admin-reindex.js';
|
||||
import { runKtxCli } from './index.js';
|
||||
|
||||
const cliVersion = (createRequire(import.meta.url)('@ktx/cli/package.json') as { version: string })
|
||||
const cliVersion = (createRequire(import.meta.url)('@kaelio/ktx/package.json') as { version: string })
|
||||
.version;
|
||||
|
||||
function makeIo(options: { stdoutIsTTY?: boolean } = {}) {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
import { KtxIngestEmbeddingPortAdapter, type KtxEmbeddingPort } from '@ktx/context';
|
||||
import { reindexLocalIndexes, type ReindexScopeResult, type ReindexSummary } from '@ktx/context/index-sync';
|
||||
import { loadKtxProject } from '@ktx/context/project';
|
||||
import { KtxIngestEmbeddingPortAdapter } from './context/llm/embedding-port.js';
|
||||
import type { KtxEmbeddingPort } from './context/core/embedding.js';
|
||||
import { reindexLocalIndexes } from './context/index-sync/reindex.js';
|
||||
import type { ReindexScopeResult, ReindexSummary } from './context/index-sync/types.js';
|
||||
import { loadKtxProject } from './context/project/project.js';
|
||||
import { Option, type Command } from '@commander-js/extra-typings';
|
||||
import { cancel, intro, log, note, outro } from '@clack/prompts';
|
||||
import type { KtxCliCommandContext } from './cli-program.js';
|
||||
|
|
@ -55,10 +57,12 @@ function quotePlainValue(value: string): string {
|
|||
return `"${value.replaceAll('\\', '\\\\').replaceAll('"', '\\"')}"`;
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export function reindexHasErrors(summary: ReindexSummary): boolean {
|
||||
return summary.scopes.some((scope) => scope.error);
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export function renderReindexPlain(summary: ReindexSummary, io: KtxCliIo): void {
|
||||
const updateKey = summary.force ? 'rebuilt' : 'updated';
|
||||
for (const scope of summary.scopes) {
|
||||
|
|
@ -88,6 +92,7 @@ export function renderReindexPlain(summary: ReindexSummary, io: KtxCliIo): void
|
|||
);
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export function renderReindexJson(summary: ReindexSummary, io: KtxCliIo): void {
|
||||
io.stdout.write(`${JSON.stringify({ kind: 'reindex', data: summary, meta: { command: 'admin reindex' } }, null, 2)}\n`);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -50,7 +50,7 @@ export function registerAdminCommands(program: Command, context: KtxCliCommandCo
|
|||
.description('Print a JSON Schema describing ktx.yaml (for editors and LLM agents)')
|
||||
.option('--output <file>', 'Write the schema to a file instead of stdout')
|
||||
.action(async (options: { output?: string }) => {
|
||||
const { generateKtxProjectConfigJsonSchema } = await import('@ktx/context/project');
|
||||
const { generateKtxProjectConfigJsonSchema } = await import('./context/project/config.js');;
|
||||
const json = `${JSON.stringify(generateKtxProjectConfigJsonSchema(), null, 2)}\n`;
|
||||
if (options.output) {
|
||||
const { writeFile } = await import('node:fs/promises');
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ export interface KtxCliPromptAdapter {
|
|||
spinner(): KtxCliSpinner;
|
||||
}
|
||||
|
||||
export class KtxCliPromptCancelledError extends Error {
|
||||
class KtxCliPromptCancelledError extends Error {
|
||||
constructor(message = 'Operation cancelled.') {
|
||||
super(message);
|
||||
this.name = 'KtxCliPromptCancelledError';
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import type { KtxProjectLlmConfig } from '@ktx/context/project';
|
||||
import type { KtxProjectLlmConfig } from './context/project/config.js';
|
||||
|
||||
const CLAUDE_CODE_IGNORED_PROMPT_CACHING_FIELDS = [
|
||||
'systemTtl',
|
||||
|
|
|
|||
|
|
@ -12,9 +12,8 @@ function stubIo(): KtxCliIo {
|
|||
|
||||
function stubPackageInfo(): KtxCliPackageInfo {
|
||||
return {
|
||||
name: '@ktx/cli',
|
||||
name: '@kaelio/ktx',
|
||||
version: '0.0.0-test',
|
||||
contextPackageName: '@ktx/context',
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,26 +0,0 @@
|
|||
import { describe, expect, it, vi } from 'vitest';
|
||||
import { buildDefaultKtxProjectConfig, type KtxLocalProject, type KtxProjectConfig } from '@ktx/context/project';
|
||||
import { loadKtxCliProject } from './cli-project.js';
|
||||
|
||||
function projectWithConfig(config: KtxProjectConfig): KtxLocalProject {
|
||||
return {
|
||||
projectDir: '/work/proj',
|
||||
configPath: '/work/proj/ktx.yaml',
|
||||
config,
|
||||
coreConfig: {} as KtxLocalProject['coreConfig'],
|
||||
git: {} as KtxLocalProject['git'],
|
||||
fileStore: {} as KtxLocalProject['fileStore'],
|
||||
};
|
||||
}
|
||||
|
||||
describe('loadKtxCliProject', () => {
|
||||
it('delegates to loadKtxProject and returns the project unchanged', async () => {
|
||||
const project = projectWithConfig(buildDefaultKtxProjectConfig());
|
||||
const loadProject = vi.fn(async () => project);
|
||||
|
||||
const result = await loadKtxCliProject({ projectDir: '/work/proj' }, { loadProject });
|
||||
|
||||
expect(result).toBe(project);
|
||||
expect(loadProject).toHaveBeenCalledWith({ projectDir: '/work/proj' });
|
||||
});
|
||||
});
|
||||
|
|
@ -1,20 +0,0 @@
|
|||
import { loadKtxProject, type KtxLocalProject } from '@ktx/context/project';
|
||||
|
||||
export interface LoadKtxCliProjectOptions {
|
||||
projectDir: string;
|
||||
}
|
||||
|
||||
export interface LoadKtxCliProjectDeps {
|
||||
loadProject?: typeof loadKtxProject;
|
||||
}
|
||||
|
||||
/**
|
||||
* Thin wrapper around `loadKtxProject`. Kept as a single entrypoint so the CLI can grow shared
|
||||
* pre-load behavior later (telemetry, project lock, etc.). Today it does no extra work.
|
||||
*/
|
||||
export async function loadKtxCliProject(
|
||||
options: LoadKtxCliProjectOptions,
|
||||
deps: LoadKtxCliProjectDeps = {},
|
||||
): Promise<KtxLocalProject> {
|
||||
return (deps.loadProject ?? loadKtxProject)({ projectDir: options.projectDir });
|
||||
}
|
||||
|
|
@ -20,7 +20,6 @@ const requirePackageJson = createRequire(import.meta.url);
|
|||
export interface KtxCliPackageInfo {
|
||||
name: string;
|
||||
version: string;
|
||||
contextPackageName: '@ktx/context';
|
||||
}
|
||||
|
||||
export interface KtxCliIo {
|
||||
|
|
@ -67,12 +66,11 @@ export function packageInfoFromJson(packageJson: unknown): KtxCliPackageInfo {
|
|||
return {
|
||||
name: packageJson.name,
|
||||
version: assertCliVersion(packageJson.version, `${packageJson.name}/package.json`),
|
||||
contextPackageName: '@ktx/context',
|
||||
};
|
||||
}
|
||||
|
||||
async function runInit(args: { projectDir: string; force: boolean }, io: KtxCliIo): Promise<number> {
|
||||
const { initKtxProject } = await import('@ktx/context/project');
|
||||
const { initKtxProject } = await import('./context/project/project.js');;
|
||||
const result = await initKtxProject({
|
||||
projectDir: args.projectDir,
|
||||
force: args.force,
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ function makeContext(overrides: Partial<KtxCliCommandContext> = {}): KtxCliComma
|
|||
stderr: { write: vi.fn() },
|
||||
},
|
||||
deps: {},
|
||||
packageInfo: { name: '@ktx/cli', version: '0.0.0-test', contextPackageName: '@ktx/context' },
|
||||
packageInfo: { name: '@kaelio/ktx', version: '0.0.0-test' },
|
||||
setExitCode: (code) => {
|
||||
exitCode = code;
|
||||
},
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ function makeContext(overrides: Partial<KtxCliCommandContext> = {}): KtxCliComma
|
|||
stderr: { write: vi.fn() },
|
||||
},
|
||||
deps: {},
|
||||
packageInfo: { name: '@ktx/cli', version: '0.0.0-test', contextPackageName: '@ktx/context' },
|
||||
packageInfo: { name: '@kaelio/ktx', version: '0.0.0-test' },
|
||||
setExitCode: (code) => {
|
||||
exitCode = code;
|
||||
},
|
||||
|
|
|
|||
|
|
@ -1,9 +1,12 @@
|
|||
import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import type { LookerClient, MetabaseRuntimeClient, NotionClient } from '@ktx/context/ingest';
|
||||
import { initKtxProject, parseKtxProjectConfig, serializeKtxProjectConfig } from '@ktx/context/project';
|
||||
import type { KtxConnectionDriver, KtxScanConnector } from '@ktx/context/scan';
|
||||
import type { LookerClient } from './context/ingest/adapters/looker/client.js';
|
||||
import type { MetabaseRuntimeClient } from './context/ingest/adapters/metabase/client-port.js';
|
||||
import type { NotionClient } from './context/ingest/adapters/notion/notion-client.js';
|
||||
import { initKtxProject } from './context/project/project.js';
|
||||
import { parseKtxProjectConfig, serializeKtxProjectConfig } from './context/project/config.js';
|
||||
import type { KtxConnectionDriver, KtxScanConnector } from './context/scan/types.js';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { runKtxConnection } from './connection.js';
|
||||
|
||||
|
|
|
|||
|
|
@ -1,19 +1,15 @@
|
|||
import {
|
||||
DEFAULT_METABASE_CLIENT_CONFIG,
|
||||
DefaultLookerConnectionClientFactory,
|
||||
DefaultMetabaseConnectionClientFactory,
|
||||
type LookerClient,
|
||||
type MetabaseRuntimeClient,
|
||||
type NotionBotInfo,
|
||||
NotionClient,
|
||||
createLocalLookerCredentialResolver,
|
||||
metabaseRuntimeConfigFromLocalConnection,
|
||||
testRepoConnection,
|
||||
} from '@ktx/context/ingest';
|
||||
import { parseNotionConnectionConfig, resolveNotionConnectionAuthToken } from '@ktx/context/connections';
|
||||
import { resolveKtxConfigReference } from '@ktx/context/core';
|
||||
import { type KtxLocalProject, loadKtxProject } from '@ktx/context/project';
|
||||
import type { KtxScanConnector } from '@ktx/context/scan';
|
||||
import { DEFAULT_METABASE_CLIENT_CONFIG, DefaultMetabaseConnectionClientFactory } from './context/ingest/adapters/metabase/client.js';
|
||||
import { DefaultLookerConnectionClientFactory } from './context/ingest/adapters/looker/factory.js';
|
||||
import type { LookerClient } from './context/ingest/adapters/looker/client.js';
|
||||
import type { MetabaseRuntimeClient } from './context/ingest/adapters/metabase/client-port.js';
|
||||
import { type NotionBotInfo, NotionClient } from './context/ingest/adapters/notion/notion-client.js';
|
||||
import { createLocalLookerCredentialResolver } from './context/ingest/adapters/looker/local-looker.adapter.js';
|
||||
import { metabaseRuntimeConfigFromLocalConnection } from './context/ingest/adapters/metabase/local-metabase.adapter.js';
|
||||
import { testRepoConnection } from './context/ingest/repo-fetch.js';
|
||||
import { parseNotionConnectionConfig, resolveNotionConnectionAuthToken } from './context/connections/notion-config.js';
|
||||
import { resolveKtxConfigReference } from './context/core/config-reference.js';
|
||||
import { type KtxLocalProject, loadKtxProject } from './context/project/project.js';
|
||||
import type { KtxScanConnector } from './context/scan/types.js';
|
||||
import type { KtxCliIo } from './index.js';
|
||||
import { bold, dim, green, red, SYMBOLS } from './io/symbols.js';
|
||||
import { createKtxCliScanConnector } from './local-scan-connectors.js';
|
||||
|
|
|
|||
|
|
@ -1,15 +1,6 @@
|
|||
import { describe, expect, it, vi } from 'vitest';
|
||||
import {
|
||||
bigQueryConnectionConfigFromConfig,
|
||||
createBigQueryLiveDatabaseIntrospection,
|
||||
isKtxBigQueryConnectionConfig,
|
||||
type KtxBigQueryClient,
|
||||
KtxBigQueryScanConnector,
|
||||
type KtxBigQueryClientFactory,
|
||||
type KtxBigQueryDataset,
|
||||
type KtxBigQueryQueryJob,
|
||||
type KtxBigQueryTableRef,
|
||||
} from './index.js';
|
||||
import { bigQueryConnectionConfigFromConfig, isKtxBigQueryConnectionConfig, type KtxBigQueryClient, KtxBigQueryScanConnector, type KtxBigQueryClientFactory, type KtxBigQueryDataset, type KtxBigQueryQueryJob, type KtxBigQueryTableRef } from '../../connectors/bigquery/connector.js';
|
||||
import { createBigQueryLiveDatabaseIntrospection } from '../../connectors/bigquery/live-database-introspection.js';
|
||||
|
||||
function fakeClientFactory(): KtxBigQueryClientFactory {
|
||||
const queryResults = vi.fn(async (): ReturnType<KtxBigQueryQueryJob['getQueryResults']> => [
|
||||
|
|
@ -1,24 +1,6 @@
|
|||
import { BigQuery, type TableField } from '@google-cloud/bigquery';
|
||||
import { assertReadOnlySql, limitSqlForExecution } from '@ktx/context/connections';
|
||||
import {
|
||||
createKtxConnectorCapabilities,
|
||||
type KtxColumnSampleInput,
|
||||
type KtxColumnSampleResult,
|
||||
type KtxColumnStatsInput,
|
||||
type KtxColumnStatsResult,
|
||||
type KtxQueryResult,
|
||||
type KtxReadOnlyQueryInput,
|
||||
type KtxScanConnector,
|
||||
type KtxScanContext,
|
||||
type KtxScanInput,
|
||||
type KtxSchemaColumn,
|
||||
type KtxSchemaSnapshot,
|
||||
type KtxSchemaTable,
|
||||
type KtxTableListEntry,
|
||||
type KtxTableRef,
|
||||
type KtxTableSampleInput,
|
||||
type KtxTableSampleResult,
|
||||
} from '@ktx/context/scan';
|
||||
import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js';
|
||||
import { createKtxConnectorCapabilities, type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaColumn, type KtxSchemaSnapshot, type KtxSchemaTable, type KtxTableListEntry, type KtxTableRef, type KtxTableSampleInput, type KtxTableSampleResult } from '../../context/scan/types.js';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { homedir } from 'node:os';
|
||||
import { resolve } from 'node:path';
|
||||
|
|
@ -57,12 +39,14 @@ export interface KtxBigQueryColumnDistinctValuesResult {
|
|||
cardinality: number;
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export interface KtxBigQueryQueryJob {
|
||||
getQueryResults(): Promise<
|
||||
[Array<Record<string, unknown>>, unknown, { schema?: { fields?: TableField[] } }?, ...unknown[]]
|
||||
>;
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export interface KtxBigQueryTableRef {
|
||||
id?: string;
|
||||
metadata?: { type?: string };
|
||||
|
|
@ -81,6 +65,7 @@ export interface KtxBigQueryTableRef {
|
|||
>;
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export interface KtxBigQueryDataset {
|
||||
get(): Promise<unknown>;
|
||||
getTables(): Promise<[KtxBigQueryTableRef[], ...unknown[]]>;
|
||||
|
|
@ -223,6 +208,7 @@ export function isKtxBigQueryConnectionConfig(
|
|||
return String(connection?.driver ?? '').toLowerCase() === 'bigquery';
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export function bigQueryConnectionConfigFromConfig(input: {
|
||||
connectionId: string;
|
||||
connection: KtxBigQueryConnectionConfig | undefined;
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
import type { KtxSchemaDimensionType, KtxTableRef } from '@ktx/context/scan';
|
||||
import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/types.js';
|
||||
|
||||
type BigQueryTableNameRef = Pick<KtxTableRef, 'name'> & Partial<Pick<KtxTableRef, 'catalog' | 'db'>>;
|
||||
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
import type { LiveDatabaseIntrospectionPort } from '@ktx/context/ingest';
|
||||
import type { KtxProjectConnectionConfig } from '@ktx/context/project';
|
||||
import type { LiveDatabaseIntrospectionPort } from '../../context/ingest/adapters/live-database/types.js';
|
||||
import type { KtxProjectConnectionConfig } from '../../context/project/config.js';
|
||||
import {
|
||||
KtxBigQueryScanConnector,
|
||||
type KtxBigQueryClientFactory,
|
||||
|
|
@ -1,11 +1,6 @@
|
|||
import { describe, expect, it, vi } from 'vitest';
|
||||
import {
|
||||
clickHouseClientConfigFromConfig,
|
||||
createClickHouseLiveDatabaseIntrospection,
|
||||
isKtxClickHouseConnectionConfig,
|
||||
KtxClickHouseScanConnector,
|
||||
type KtxClickHouseClientFactory,
|
||||
} from './index.js';
|
||||
import { clickHouseClientConfigFromConfig, isKtxClickHouseConnectionConfig, KtxClickHouseScanConnector, type KtxClickHouseClientFactory } from '../../connectors/clickhouse/connector.js';
|
||||
import { createClickHouseLiveDatabaseIntrospection } from '../../connectors/clickhouse/live-database-introspection.js';
|
||||
|
||||
function result<T>(payload: T) {
|
||||
return {
|
||||
|
|
@ -1,24 +1,6 @@
|
|||
import { createClient } from '@clickhouse/client';
|
||||
import { assertReadOnlySql, limitSqlForExecution } from '@ktx/context/connections';
|
||||
import {
|
||||
createKtxConnectorCapabilities,
|
||||
type KtxColumnSampleInput,
|
||||
type KtxColumnSampleResult,
|
||||
type KtxColumnStatsInput,
|
||||
type KtxColumnStatsResult,
|
||||
type KtxQueryResult,
|
||||
type KtxReadOnlyQueryInput,
|
||||
type KtxScanConnector,
|
||||
type KtxScanContext,
|
||||
type KtxScanInput,
|
||||
type KtxSchemaColumn,
|
||||
type KtxSchemaSnapshot,
|
||||
type KtxSchemaTable,
|
||||
type KtxTableRef,
|
||||
type KtxTableSampleInput,
|
||||
type KtxTableListEntry,
|
||||
type KtxTableSampleResult,
|
||||
} from '@ktx/context/scan';
|
||||
import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js';
|
||||
import { createKtxConnectorCapabilities, type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaColumn, type KtxSchemaSnapshot, type KtxSchemaTable, type KtxTableRef, type KtxTableSampleInput, type KtxTableListEntry, type KtxTableSampleResult } from '../../context/scan/types.js';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { Agent as HttpsAgent } from 'node:https';
|
||||
import { homedir } from 'node:os';
|
||||
|
|
@ -198,6 +180,7 @@ export function isKtxClickHouseConnectionConfig(
|
|||
return String(connection?.driver ?? '').toLowerCase() === 'clickhouse';
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export function clickHouseClientConfigFromConfig(input: {
|
||||
connectionId: string;
|
||||
connection: KtxClickHouseConnectionConfig | undefined;
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
import type { KtxSchemaDimensionType, KtxTableRef } from '@ktx/context/scan';
|
||||
import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/types.js';
|
||||
|
||||
type ClickHouseTableNameRef = Pick<KtxTableRef, 'name'> & Partial<Pick<KtxTableRef, 'catalog' | 'db'>>;
|
||||
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
import type { LiveDatabaseIntrospectionPort } from '@ktx/context/ingest';
|
||||
import type { KtxProjectConnectionConfig } from '@ktx/context/project';
|
||||
import type { LiveDatabaseIntrospectionPort } from '../../context/ingest/adapters/live-database/types.js';
|
||||
import type { KtxProjectConnectionConfig } from '../../context/project/config.js';
|
||||
import {
|
||||
KtxClickHouseScanConnector,
|
||||
type KtxClickHouseClientFactory,
|
||||
|
|
@ -1,12 +1,7 @@
|
|||
import { describe, expect, it, vi } from 'vitest';
|
||||
import type { FieldPacket, RowDataPacket } from 'mysql2/promise';
|
||||
import {
|
||||
createMysqlLiveDatabaseIntrospection,
|
||||
isKtxMysqlConnectionConfig,
|
||||
KtxMysqlScanConnector,
|
||||
mysqlConnectionPoolConfigFromConfig,
|
||||
type KtxMysqlPoolFactory,
|
||||
} from './index.js';
|
||||
import { createMysqlLiveDatabaseIntrospection } from '../../connectors/mysql/live-database-introspection.js';
|
||||
import { isKtxMysqlConnectionConfig, KtxMysqlScanConnector, mysqlConnectionPoolConfigFromConfig, type KtxMysqlPoolFactory } from '../../connectors/mysql/connector.js';
|
||||
|
||||
function mysqlResult(rows: Record<string, unknown>[], fields: Array<{ name: string; type?: number }>): [RowDataPacket[], FieldPacket[]] {
|
||||
return [rows as RowDataPacket[], fields as FieldPacket[]];
|
||||
|
|
@ -2,27 +2,8 @@ import mysql, { type FieldPacket, type Pool, type RowDataPacket } from 'mysql2/p
|
|||
import { readFileSync } from 'node:fs';
|
||||
import { homedir } from 'node:os';
|
||||
import { resolve } from 'node:path';
|
||||
import { assertReadOnlySql, limitSqlForExecution } from '@ktx/context/connections';
|
||||
import {
|
||||
createKtxConnectorCapabilities,
|
||||
type KtxColumnSampleInput,
|
||||
type KtxColumnSampleResult,
|
||||
type KtxColumnStatsInput,
|
||||
type KtxColumnStatsResult,
|
||||
type KtxQueryResult,
|
||||
type KtxReadOnlyQueryInput,
|
||||
type KtxScanConnector,
|
||||
type KtxScanContext,
|
||||
type KtxScanInput,
|
||||
type KtxSchemaColumn,
|
||||
type KtxTableListEntry,
|
||||
type KtxSchemaForeignKey,
|
||||
type KtxSchemaSnapshot,
|
||||
type KtxSchemaTable,
|
||||
type KtxTableRef,
|
||||
type KtxTableSampleInput,
|
||||
type KtxTableSampleResult,
|
||||
} from '@ktx/context/scan';
|
||||
import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js';
|
||||
import { createKtxConnectorCapabilities, type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaColumn, type KtxTableListEntry, type KtxSchemaForeignKey, type KtxSchemaSnapshot, type KtxSchemaTable, type KtxTableRef, type KtxTableSampleInput, type KtxTableSampleResult } from '../../context/scan/types.js';
|
||||
import { KtxMysqlDialect } from './dialect.js';
|
||||
|
||||
export interface KtxMysqlConnectionConfig {
|
||||
|
|
@ -237,6 +218,7 @@ export function isKtxMysqlConnectionConfig(
|
|||
return String(connection?.driver ?? '').toLowerCase() === 'mysql';
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export function mysqlConnectionPoolConfigFromConfig(input: {
|
||||
connectionId: string;
|
||||
connection: KtxMysqlConnectionConfig | undefined;
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
import type { KtxSchemaDimensionType, KtxTableRef } from '@ktx/context/scan';
|
||||
import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/types.js';
|
||||
|
||||
type MysqlTableNameRef = Pick<KtxTableRef, 'name'> & Partial<Pick<KtxTableRef, 'catalog' | 'db'>>;
|
||||
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
import type { LiveDatabaseIntrospectionPort } from '@ktx/context/ingest';
|
||||
import type { KtxProjectConnectionConfig } from '@ktx/context/project';
|
||||
import type { LiveDatabaseIntrospectionPort } from '../../context/ingest/adapters/live-database/types.js';
|
||||
import type { KtxProjectConnectionConfig } from '../../context/project/config.js';
|
||||
import {
|
||||
KtxMysqlScanConnector,
|
||||
type KtxMysqlConnectionConfig,
|
||||
|
|
@ -1,11 +1,6 @@
|
|||
import { describe, expect, it, vi } from 'vitest';
|
||||
import {
|
||||
createPostgresLiveDatabaseIntrospection,
|
||||
isKtxPostgresConnectionConfig,
|
||||
KtxPostgresScanConnector,
|
||||
postgresPoolConfigFromConfig,
|
||||
type KtxPostgresPoolFactory,
|
||||
} from './index.js';
|
||||
import { createPostgresLiveDatabaseIntrospection } from '../../connectors/postgres/live-database-introspection.js';
|
||||
import { isKtxPostgresConnectionConfig, KtxPostgresScanConnector, postgresPoolConfigFromConfig, type KtxPostgresPoolFactory } from '../../connectors/postgres/connector.js';
|
||||
|
||||
interface FakeQueryResult {
|
||||
rows: Record<string, unknown>[];
|
||||
|
|
@ -1,27 +1,8 @@
|
|||
import { readFileSync } from 'node:fs';
|
||||
import { homedir } from 'node:os';
|
||||
import { resolve } from 'node:path';
|
||||
import { assertReadOnlySql, limitSqlForExecution } from '@ktx/context/connections';
|
||||
import {
|
||||
createKtxConnectorCapabilities,
|
||||
type KtxColumnSampleInput,
|
||||
type KtxColumnSampleResult,
|
||||
type KtxColumnStatsInput,
|
||||
type KtxColumnStatsResult,
|
||||
type KtxQueryResult,
|
||||
type KtxReadOnlyQueryInput,
|
||||
type KtxScanConnector,
|
||||
type KtxScanContext,
|
||||
type KtxScanInput,
|
||||
type KtxSchemaColumn,
|
||||
type KtxSchemaForeignKey,
|
||||
type KtxSchemaSnapshot,
|
||||
type KtxSchemaTable,
|
||||
type KtxTableListEntry,
|
||||
type KtxTableRef,
|
||||
type KtxTableSampleInput,
|
||||
type KtxTableSampleResult,
|
||||
} from '@ktx/context/scan';
|
||||
import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js';
|
||||
import { createKtxConnectorCapabilities, type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaColumn, type KtxSchemaForeignKey, type KtxSchemaSnapshot, type KtxSchemaTable, type KtxTableListEntry, type KtxTableRef, type KtxTableSampleInput, type KtxTableSampleResult } from '../../context/scan/types.js';
|
||||
import { Pool } from 'pg';
|
||||
import { KtxPostgresDialect } from './dialect.js';
|
||||
|
||||
|
|
@ -297,6 +278,7 @@ export function isKtxPostgresConnectionConfig(
|
|||
return driver === 'postgres' || driver === 'postgresql';
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export function postgresPoolConfigFromConfig(input: {
|
||||
connectionId: string;
|
||||
connection: KtxPostgresConnectionConfig | undefined;
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
import type { KtxSchemaDimensionType, KtxTableRef } from '@ktx/context/scan';
|
||||
import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/types.js';
|
||||
|
||||
type PostgresTableNameRef = Pick<KtxTableRef, 'name'> & Partial<Pick<KtxTableRef, 'catalog' | 'db'>>;
|
||||
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
import type { KtxPostgresQueryClient } from '@ktx/context/ingest';
|
||||
import type { KtxPostgresQueryClient } from '../../context/ingest/adapters/historic-sql/types.js';
|
||||
import { KtxPostgresScanConnector, type KtxPostgresScanConnectorOptions } from './connector.js';
|
||||
|
||||
export type KtxPostgresHistoricSqlQueryClientOptions = KtxPostgresScanConnectorOptions;
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
import type { LiveDatabaseIntrospectionPort } from '@ktx/context/ingest';
|
||||
import type { KtxProjectConnectionConfig } from '@ktx/context/project';
|
||||
import type { LiveDatabaseIntrospectionPort } from '../../context/ingest/adapters/live-database/types.js';
|
||||
import type { KtxProjectConnectionConfig } from '../../context/project/config.js';
|
||||
import {
|
||||
KtxPostgresScanConnector,
|
||||
type KtxPostgresConnectionConfig,
|
||||
|
|
@ -1,12 +1,6 @@
|
|||
import { describe, expect, it, vi } from 'vitest';
|
||||
import {
|
||||
createSnowflakeLiveDatabaseIntrospection,
|
||||
isKtxSnowflakeConnectionConfig,
|
||||
KtxSnowflakeScanConnector,
|
||||
snowflakeConnectionConfigFromConfig,
|
||||
type KtxSnowflakeDriver,
|
||||
type KtxSnowflakeDriverFactory,
|
||||
} from './index.js';
|
||||
import { createSnowflakeLiveDatabaseIntrospection } from '../../connectors/snowflake/live-database-introspection.js';
|
||||
import { isKtxSnowflakeConnectionConfig, KtxSnowflakeScanConnector, snowflakeConnectionConfigFromConfig, type KtxSnowflakeDriver, type KtxSnowflakeDriverFactory } from '../../connectors/snowflake/connector.js';
|
||||
|
||||
function fakeDriverFactory(): KtxSnowflakeDriverFactory {
|
||||
const driver: KtxSnowflakeDriver = {
|
||||
|
|
@ -2,26 +2,8 @@ import { createPrivateKey } from 'node:crypto';
|
|||
import { readFileSync } from 'node:fs';
|
||||
import { homedir } from 'node:os';
|
||||
import { resolve } from 'node:path';
|
||||
import { assertReadOnlySql, limitSqlForExecution } from '@ktx/context/connections';
|
||||
import {
|
||||
createKtxConnectorCapabilities,
|
||||
type KtxColumnSampleInput,
|
||||
type KtxColumnSampleResult,
|
||||
type KtxColumnStatsInput,
|
||||
type KtxColumnStatsResult,
|
||||
type KtxQueryResult,
|
||||
type KtxReadOnlyQueryInput,
|
||||
type KtxScanConnector,
|
||||
type KtxScanContext,
|
||||
type KtxScanInput,
|
||||
type KtxSchemaColumn,
|
||||
type KtxSchemaSnapshot,
|
||||
type KtxSchemaTable,
|
||||
type KtxTableRef,
|
||||
type KtxTableSampleInput,
|
||||
type KtxTableListEntry,
|
||||
type KtxTableSampleResult,
|
||||
} from '@ktx/context/scan';
|
||||
import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js';
|
||||
import { createKtxConnectorCapabilities, type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaColumn, type KtxSchemaSnapshot, type KtxSchemaTable, type KtxTableRef, type KtxTableSampleInput, type KtxTableListEntry, type KtxTableSampleResult } from '../../context/scan/types.js';
|
||||
import * as snowflake from 'snowflake-sdk';
|
||||
import { KtxSnowflakeDialect } from './dialect.js';
|
||||
|
||||
|
|
@ -196,6 +178,7 @@ export function isKtxSnowflakeConnectionConfig(
|
|||
return String(connection?.driver ?? '').toLowerCase() === 'snowflake';
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export function snowflakeConnectionConfigFromConfig(input: {
|
||||
connectionId: string;
|
||||
connection: KtxSnowflakeConnectionConfig | undefined;
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
import type { KtxSchemaDimensionType, KtxTableRef } from '@ktx/context/scan';
|
||||
import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/types.js';
|
||||
|
||||
type SnowflakeTableNameRef = Pick<KtxTableRef, 'name'> & Partial<Pick<KtxTableRef, 'catalog' | 'db'>>;
|
||||
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
import type { LiveDatabaseIntrospectionPort } from '@ktx/context/ingest';
|
||||
import type { KtxProjectConnectionConfig } from '@ktx/context/project';
|
||||
import type { LiveDatabaseIntrospectionPort } from '../../context/ingest/adapters/live-database/types.js';
|
||||
import type { KtxProjectConnectionConfig } from '../../context/project/config.js';
|
||||
import {
|
||||
KtxSnowflakeScanConnector,
|
||||
type KtxSnowflakeConnectionConfig,
|
||||
|
|
@ -4,12 +4,8 @@ import { mkdtemp, rm } from 'node:fs/promises';
|
|||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
||||
import {
|
||||
createSqliteLiveDatabaseIntrospection,
|
||||
isKtxSqliteConnectionConfig,
|
||||
KtxSqliteScanConnector,
|
||||
sqliteDatabasePathFromConfig,
|
||||
} from './index.js';
|
||||
import { createSqliteLiveDatabaseIntrospection } from '../../connectors/sqlite/live-database-introspection.js';
|
||||
import { isKtxSqliteConnectionConfig, KtxSqliteScanConnector, sqliteDatabasePathFromConfig } from '../../connectors/sqlite/connector.js';
|
||||
|
||||
describe('KtxSqliteScanConnector', () => {
|
||||
let tempDir: string;
|
||||
|
|
@ -3,25 +3,9 @@ import { existsSync, readFileSync, statSync } from 'node:fs';
|
|||
import { homedir } from 'node:os';
|
||||
import { isAbsolute, resolve } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { assertReadOnlySql, limitSqlForExecution, normalizeQueryRows } from '@ktx/context/connections';
|
||||
import {
|
||||
createKtxConnectorCapabilities,
|
||||
type KtxColumnSampleInput,
|
||||
type KtxColumnSampleResult,
|
||||
type KtxColumnStatsInput,
|
||||
type KtxColumnStatsResult,
|
||||
type KtxQueryResult,
|
||||
type KtxReadOnlyQueryInput,
|
||||
type KtxScanConnector,
|
||||
type KtxScanContext,
|
||||
type KtxScanInput,
|
||||
type KtxSchemaForeignKey,
|
||||
type KtxSchemaSnapshot,
|
||||
type KtxSchemaTable,
|
||||
type KtxTableRef,
|
||||
type KtxTableSampleInput,
|
||||
type KtxTableSampleResult,
|
||||
} from '@ktx/context/scan';
|
||||
import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js';
|
||||
import { normalizeQueryRows } from '../../context/connections/query-executor.js';
|
||||
import { createKtxConnectorCapabilities, type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaForeignKey, type KtxSchemaSnapshot, type KtxSchemaTable, type KtxTableRef, type KtxTableSampleInput, type KtxTableSampleResult } from '../../context/scan/types.js';
|
||||
import { KtxSqliteDialect } from './dialect.js';
|
||||
|
||||
export interface KtxSqliteConnectionConfig {
|
||||
|
|
@ -31,6 +15,7 @@ export interface KtxSqliteConnectionConfig {
|
|||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export interface SqliteDatabasePathInput {
|
||||
connectionId: string;
|
||||
projectDir?: string;
|
||||
|
|
@ -142,6 +127,7 @@ export function isKtxSqliteConnectionConfig(
|
|||
return driver === 'sqlite' || driver === 'sqlite3';
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export function sqliteDatabasePathFromConfig(input: SqliteDatabasePathInput): string {
|
||||
const inputDriver = input.connection?.driver ?? 'unknown';
|
||||
if (!isKtxSqliteConnectionConfig(input.connection)) {
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
import type { KtxSchemaDimensionType, KtxTableRef } from '@ktx/context/scan';
|
||||
import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/types.js';
|
||||
|
||||
type SqliteTableNameRef = Pick<KtxTableRef, 'name'> & Partial<Pick<KtxTableRef, 'catalog' | 'db'>>;
|
||||
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
import type { LiveDatabaseIntrospectionPort } from '@ktx/context/ingest';
|
||||
import type { KtxProjectConnectionConfig } from '@ktx/context/project';
|
||||
import type { LiveDatabaseIntrospectionPort } from '../../context/ingest/adapters/live-database/types.js';
|
||||
import type { KtxProjectConnectionConfig } from '../../context/project/config.js';
|
||||
import { KtxSqliteScanConnector, type KtxSqliteConnectionConfig } from './connector.js';
|
||||
|
||||
export interface CreateSqliteLiveDatabaseIntrospectionOptions {
|
||||
|
|
@ -1,12 +1,6 @@
|
|||
import { describe, expect, it, vi } from 'vitest';
|
||||
import {
|
||||
createSqlServerLiveDatabaseIntrospection,
|
||||
isKtxSqlServerConnectionConfig,
|
||||
KtxSqlServerScanConnector,
|
||||
sqlServerConnectionPoolConfigFromConfig,
|
||||
type KtxSqlServerPoolFactory,
|
||||
type KtxSqlServerQueryResult,
|
||||
} from './index.js';
|
||||
import { createSqlServerLiveDatabaseIntrospection } from '../../connectors/sqlserver/live-database-introspection.js';
|
||||
import { isKtxSqlServerConnectionConfig, KtxSqlServerScanConnector, sqlServerConnectionPoolConfigFromConfig, type KtxSqlServerPoolFactory, type KtxSqlServerQueryResult } from '../../connectors/sqlserver/connector.js';
|
||||
|
||||
function recordset<T extends Record<string, unknown>>(
|
||||
rows: T[],
|
||||
|
|
@ -1,24 +1,5 @@
|
|||
import { assertReadOnlySql } from '@ktx/context/connections';
|
||||
import {
|
||||
createKtxConnectorCapabilities,
|
||||
type KtxColumnSampleInput,
|
||||
type KtxColumnSampleResult,
|
||||
type KtxColumnStatsInput,
|
||||
type KtxColumnStatsResult,
|
||||
type KtxQueryResult,
|
||||
type KtxReadOnlyQueryInput,
|
||||
type KtxScanConnector,
|
||||
type KtxScanContext,
|
||||
type KtxScanInput,
|
||||
type KtxSchemaColumn,
|
||||
type KtxSchemaForeignKey,
|
||||
type KtxSchemaSnapshot,
|
||||
type KtxSchemaTable,
|
||||
type KtxTableListEntry,
|
||||
type KtxTableRef,
|
||||
type KtxTableSampleInput,
|
||||
type KtxTableSampleResult,
|
||||
} from '@ktx/context/scan';
|
||||
import { assertReadOnlySql } from '../../context/connections/read-only-sql.js';
|
||||
import { createKtxConnectorCapabilities, type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaColumn, type KtxSchemaForeignKey, type KtxSchemaSnapshot, type KtxSchemaTable, type KtxTableListEntry, type KtxTableRef, type KtxTableSampleInput, type KtxTableSampleResult } from '../../context/scan/types.js';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { homedir } from 'node:os';
|
||||
import { resolve } from 'node:path';
|
||||
|
|
@ -50,6 +31,7 @@ export interface KtxSqlServerPoolConfig {
|
|||
pool: { max: number; min: number; idleTimeoutMillis: number };
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export interface KtxSqlServerQueryResult {
|
||||
recordset?: Array<Record<string, unknown>> & { columns?: Record<string, { type?: { declaration?: string } }> };
|
||||
}
|
||||
|
|
@ -239,6 +221,7 @@ export function isKtxSqlServerConnectionConfig(
|
|||
return String(connection?.driver ?? '').toLowerCase() === 'sqlserver';
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export function sqlServerConnectionPoolConfigFromConfig(input: {
|
||||
connectionId: string;
|
||||
connection: KtxSqlServerConnectionConfig | undefined;
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
import type { KtxSchemaDimensionType, KtxTableRef } from '@ktx/context/scan';
|
||||
import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/types.js';
|
||||
|
||||
type SqlServerTableNameRef = Pick<KtxTableRef, 'name'> & Partial<Pick<KtxTableRef, 'catalog' | 'db'>>;
|
||||
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
import type { LiveDatabaseIntrospectionPort } from '@ktx/context/ingest';
|
||||
import type { KtxProjectConnectionConfig } from '@ktx/context/project';
|
||||
import type { LiveDatabaseIntrospectionPort } from '../../context/ingest/adapters/live-database/types.js';
|
||||
import type { KtxProjectConnectionConfig } from '../../context/project/config.js';
|
||||
import {
|
||||
KtxSqlServerScanConnector,
|
||||
type KtxSqlServerConnectionConfig,
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
import { buildDefaultKtxProjectConfig, type KtxProjectConfig } from '@ktx/context/project';
|
||||
import { buildDefaultKtxProjectConfig, type KtxProjectConfig } from './context/project/config.js';
|
||||
import { describe, expect, it, vi } from 'vitest';
|
||||
import type { KtxPublicIngestProject, KtxPublicIngestTargetResult } from './public-ingest.js';
|
||||
import {
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import type { KtxProgressPort, KtxProgressUpdateOptions } from '@ktx/context/scan';
|
||||
import type { KtxProgressPort, KtxProgressUpdateOptions } from './context/scan/types.js';
|
||||
import type { KtxCliIo } from './index.js';
|
||||
import type { KtxIngestProgressUpdate } from './ingest.js';
|
||||
import type { KtxManagedPythonInstallPolicy } from './managed-python-command.js';
|
||||
|
|
@ -444,17 +444,20 @@ export function renderContextBuildView(
|
|||
const ESC_K_RE = new RegExp(`${ESC.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\[K`, 'g');
|
||||
const ANSI_RE = /\x1b\[[0-9;]*m/g;
|
||||
|
||||
/** @internal */
|
||||
export function extractProgressMessage(chunk: string): string | null {
|
||||
const cleaned = chunk.replace(/^\r/, '').replace(ESC_K_RE, '').replace(/\n$/, '').trim();
|
||||
const match = cleaned.match(/^\[(\d+)%\]\s*(.+)$/);
|
||||
return match ? `[${match[1]}%] ${match[2]}` : null;
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export function parseScanSummary(output: string): string | null {
|
||||
const match = output.match(/(\d+) changes? across (\d+) tables?/);
|
||||
return match ? `${match[2]} tables` : null;
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export function parseIngestSummary(output: string): string | null {
|
||||
const savedMemory = output.match(/Saved memory: (.+)/);
|
||||
if (savedMemory) return savedMemory[1];
|
||||
|
|
@ -560,6 +563,7 @@ function collectSourceProgress(targets: ContextBuildTargetState[]): ContextBuild
|
|||
});
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export function viewStateFromSourceProgress(
|
||||
sources: ContextBuildSourceProgressUpdate[],
|
||||
now: number,
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import type { KtxSchemaDimensionType, KtxTableRef } from '../scan/types.js';
|
||||
|
||||
export type SupportedDriver =
|
||||
type SupportedDriver =
|
||||
| 'postgres'
|
||||
| 'postgresql'
|
||||
| 'mysql'
|
||||
|
|
@ -8,7 +8,7 @@ import {
|
|||
} from '../ingest/adapters/notion/types.js';
|
||||
import type { KtxProjectConnectionConfig } from '../project/config.js';
|
||||
|
||||
export const KTX_NOTION_ORG_KNOWLEDGE_WARNING =
|
||||
const KTX_NOTION_ORG_KNOWLEDGE_WARNING =
|
||||
'Anything accessible to this Notion integration can become organization knowledge.';
|
||||
|
||||
type KtxNotionCrawlMode = 'all_accessible' | 'selected_roots';
|
||||
|
|
@ -39,6 +39,7 @@ export type KtxNotionConnectionConfig = Omit<
|
|||
max_knowledge_updates_per_run: number;
|
||||
};
|
||||
|
||||
/** @internal */
|
||||
export interface RedactedKtxNotionConnectionConfig {
|
||||
driver: 'notion';
|
||||
hasAuthToken: boolean;
|
||||
|
|
@ -152,6 +153,7 @@ export function parseNotionConnectionConfig(raw: unknown): KtxNotionConnectionCo
|
|||
};
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export function redactNotionConnectionConfig(config: KtxNotionConnectionConfig): RedactedKtxNotionConnectionConfig {
|
||||
return {
|
||||
driver: 'notion',
|
||||
|
|
@ -171,6 +173,7 @@ function expandHome(path: string): string {
|
|||
return path === '~' || path.startsWith('~/') ? resolve(homedir(), path.slice(2)) : path;
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export async function resolveNotionAuthToken(
|
||||
authTokenRef: string,
|
||||
options: ResolveNotionTokenOptions = {},
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
import type { KtxProjectConnectionConfig } from '../project/index.js';
|
||||
import type { KtxProjectConnectionConfig } from '../../context/project/config.js';
|
||||
|
||||
export interface KtxSqlQueryExecutionInput {
|
||||
connectionId: string;
|
||||
|
|
@ -49,6 +49,7 @@ function sqlitePathFromUrl(url: string): string {
|
|||
return url;
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export function sqliteDatabasePathFromConnection(input: KtxSqlQueryExecutionInput): string {
|
||||
const driver = connectionDriver(input);
|
||||
if (driver !== 'sqlite' && driver !== 'sqlite3') {
|
||||
|
|
@ -2,6 +2,7 @@ import { readFileSync } from 'node:fs';
|
|||
import { homedir } from 'node:os';
|
||||
import { resolve } from 'node:path';
|
||||
|
||||
/** @internal */
|
||||
export function resolveKtxHomePath(path: string): string {
|
||||
if (path === '~') {
|
||||
return homedir();
|
||||
|
|
@ -1,10 +1,10 @@
|
|||
export interface KtxStorageConfig {
|
||||
interface KtxStorageConfig {
|
||||
configDir?: string;
|
||||
homeDir?: string;
|
||||
worktreesDir?: string;
|
||||
}
|
||||
|
||||
export interface KtxGitConfig {
|
||||
interface KtxGitConfig {
|
||||
userName: string;
|
||||
userEmail: string;
|
||||
bootstrapMessage?: string;
|
||||
|
|
@ -1,3 +1,4 @@
|
|||
/** @internal */
|
||||
export const REDACTED_KTX_CREDENTIAL_VALUE = '<redacted>';
|
||||
|
||||
const SENSITIVE_FIELD_NAME = /(password|secret|token|api[_-]?key|private[_-]?key|passphrase|credential|authorization|url)/i;
|
||||
|
|
@ -5,7 +5,7 @@ import { GitService } from './git.service.js';
|
|||
|
||||
export type SessionOutcome = 'success' | 'empty' | 'conflict' | 'crash';
|
||||
|
||||
export interface SentinelPayload {
|
||||
interface SentinelPayload {
|
||||
outcome: SessionOutcome;
|
||||
at: string;
|
||||
chatId: string;
|
||||
|
|
@ -4,21 +4,21 @@ import { URL } from 'node:url';
|
|||
import { spawn } from 'node:child_process';
|
||||
import type { ResolvedSemanticLayerSource, SemanticLayerQueryInput } from '../sl/types.js';
|
||||
|
||||
export interface KtxSemanticLayerComputeQueryResult {
|
||||
interface KtxSemanticLayerComputeQueryResult {
|
||||
sql: string;
|
||||
dialect: string;
|
||||
columns: Array<Record<string, unknown>>;
|
||||
plan: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface KtxSemanticLayerComputeValidationResult {
|
||||
interface KtxSemanticLayerComputeValidationResult {
|
||||
valid: boolean;
|
||||
errors: string[];
|
||||
warnings: string[];
|
||||
perSourceWarnings: Record<string, string[]>;
|
||||
}
|
||||
|
||||
export interface KtxSemanticLayerSourceGenerationColumnInput {
|
||||
interface KtxSemanticLayerSourceGenerationColumnInput {
|
||||
name: string;
|
||||
type: string;
|
||||
primaryKey?: boolean;
|
||||
|
|
@ -26,7 +26,7 @@ export interface KtxSemanticLayerSourceGenerationColumnInput {
|
|||
comment?: string | null;
|
||||
}
|
||||
|
||||
export interface KtxSemanticLayerSourceGenerationTableInput {
|
||||
interface KtxSemanticLayerSourceGenerationTableInput {
|
||||
name: string;
|
||||
catalog?: string | null;
|
||||
db?: string | null;
|
||||
|
|
@ -34,7 +34,7 @@ export interface KtxSemanticLayerSourceGenerationTableInput {
|
|||
columns: KtxSemanticLayerSourceGenerationColumnInput[];
|
||||
}
|
||||
|
||||
export interface KtxSemanticLayerSourceGenerationLinkInput {
|
||||
interface KtxSemanticLayerSourceGenerationLinkInput {
|
||||
fromTable: string;
|
||||
fromColumn: string;
|
||||
toTable: string;
|
||||
|
|
@ -42,13 +42,13 @@ export interface KtxSemanticLayerSourceGenerationLinkInput {
|
|||
relationshipType: string;
|
||||
}
|
||||
|
||||
export interface KtxSemanticLayerSourceGenerationInput {
|
||||
interface KtxSemanticLayerSourceGenerationInput {
|
||||
tables: KtxSemanticLayerSourceGenerationTableInput[];
|
||||
links: KtxSemanticLayerSourceGenerationLinkInput[];
|
||||
dialect?: string;
|
||||
}
|
||||
|
||||
export interface KtxSemanticLayerSourceGenerationResult {
|
||||
interface KtxSemanticLayerSourceGenerationResult {
|
||||
sources: Array<Record<string, unknown>>;
|
||||
sourceCount: number;
|
||||
}
|
||||
|
|
@ -75,14 +75,14 @@ export interface KtxSemanticLayerComputePort {
|
|||
generateSources(input: KtxSemanticLayerSourceGenerationInput): Promise<KtxSemanticLayerSourceGenerationResult>;
|
||||
}
|
||||
|
||||
export type KtxDaemonCommand = 'semantic-query' | 'semantic-validate' | 'semantic-generate-sources';
|
||||
type KtxDaemonCommand = 'semantic-query' | 'semantic-validate' | 'semantic-generate-sources';
|
||||
|
||||
export type KtxDaemonJsonRunner = (
|
||||
type KtxDaemonJsonRunner = (
|
||||
subcommand: KtxDaemonCommand,
|
||||
payload: Record<string, unknown>,
|
||||
) => Promise<Record<string, unknown>>;
|
||||
|
||||
export type KtxDaemonHttpJsonRunner = (path: string, payload: Record<string, unknown>) => Promise<Record<string, unknown>>;
|
||||
type KtxDaemonHttpJsonRunner = (path: string, payload: Record<string, unknown>) => Promise<Record<string, unknown>>;
|
||||
|
||||
export interface PythonSemanticLayerComputeOptions {
|
||||
command?: string;
|
||||
|
|
@ -92,6 +92,7 @@ export interface PythonSemanticLayerComputeOptions {
|
|||
runJson?: KtxDaemonJsonRunner;
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export interface HttpSemanticLayerComputeOptions {
|
||||
baseUrl: string;
|
||||
requestJson?: KtxDaemonHttpJsonRunner;
|
||||
|
|
@ -272,6 +273,7 @@ export function createPythonSemanticLayerComputePort(
|
|||
};
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export function createHttpSemanticLayerComputePort(
|
||||
options: HttpSemanticLayerComputeOptions,
|
||||
): KtxSemanticLayerComputePort {
|
||||
|
|
@ -2,8 +2,8 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
|
|||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
||||
import type { KtxEmbeddingPort } from '../core/index.js';
|
||||
import { initKtxProject, loadKtxProject, type KtxLocalProject } from '../project/index.js';
|
||||
import type { KtxEmbeddingPort } from '../../context/core/embedding.js';
|
||||
import { initKtxProject, loadKtxProject, type KtxLocalProject } from '../../context/project/project.js';
|
||||
import { SqliteKnowledgeIndex } from '../wiki/sqlite-knowledge-index.js';
|
||||
import { reindexLocalIndexes } from './reindex.js';
|
||||
|
||||
|
|
@ -1,8 +1,12 @@
|
|||
import { readdir, stat } from 'node:fs/promises';
|
||||
import { join, relative } from 'node:path';
|
||||
import { ktxLocalStateDbPath, type KtxLocalProject } from '../project/index.js';
|
||||
import { loadLocalSlSourceRecords, SlSearchService, SqliteSlSourcesIndex } from '../sl/index.js';
|
||||
import { KnowledgeWikiService, SqliteKnowledgeIndex } from '../wiki/index.js';
|
||||
import { ktxLocalStateDbPath } from '../../context/project/local-state-db.js';
|
||||
import type { KtxLocalProject } from '../../context/project/project.js';
|
||||
import { loadLocalSlSourceRecords } from '../../context/sl/local-sl.js';
|
||||
import { SlSearchService } from '../../context/sl/sl-search.service.js';
|
||||
import { SqliteSlSourcesIndex } from '../../context/sl/sqlite-sl-sources-index.js';
|
||||
import { KnowledgeWikiService } from '../../context/wiki/knowledge-wiki.service.js';
|
||||
import { SqliteKnowledgeIndex } from '../../context/wiki/sqlite-knowledge-index.js';
|
||||
import type { ReindexOptions, ReindexScopeResult, ReindexSummary, ReindexWorkResult } from './types.js';
|
||||
|
||||
type DiscoveredScope =
|
||||
|
|
@ -41,7 +45,7 @@ async function childDirectories(path: string): Promise<string[]> {
|
|||
}
|
||||
}
|
||||
|
||||
export async function discoverReindexScopes(project: KtxLocalProject): Promise<DiscoveredScope[]> {
|
||||
async function discoverReindexScopes(project: KtxLocalProject): Promise<DiscoveredScope[]> {
|
||||
const scopes: DiscoveredScope[] = [];
|
||||
if (await directoryExists(join(project.projectDir, 'wiki/global'))) {
|
||||
scopes.push({ kind: 'wiki', scope: 'GLOBAL', scopeId: null, label: 'global' });
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
import type { KtxEmbeddingPort } from '../core/index.js';
|
||||
import type { KtxEmbeddingPort } from '../../context/core/embedding.js';
|
||||
|
||||
export interface ReindexOptions {
|
||||
force: boolean;
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
import type { MemoryAction } from '../memory/index.js';
|
||||
import type { MemoryAction } from '../../context/memory/types.js';
|
||||
|
||||
export function actionTargetConnectionId(action: MemoryAction, runConnectionId: string): string {
|
||||
return action.target === 'sl' ? (action.targetConnectionId ?? runConnectionId) : runConnectionId;
|
||||
|
|
@ -1,9 +1,9 @@
|
|||
import { createHash } from 'node:crypto';
|
||||
import { parse as parseYaml } from 'yaml';
|
||||
import { type KtxLogger, noopLogger } from '../../../core/index.js';
|
||||
import { type KtxLogger, noopLogger } from '../../../../context/core/config.js';
|
||||
import { resolveJinjaVariables } from '../../dbt-shared/project-vars.js';
|
||||
|
||||
export interface DbtParsedColumn {
|
||||
interface DbtParsedColumn {
|
||||
name: string;
|
||||
description: string | null;
|
||||
dataType: string | null;
|
||||
|
|
@ -12,20 +12,20 @@ export interface DbtParsedColumn {
|
|||
enumValuesDbt?: string[];
|
||||
}
|
||||
|
||||
export interface DbtDataTestRef {
|
||||
interface DbtDataTestRef {
|
||||
name: string;
|
||||
package: string;
|
||||
kwargs?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface DbtColumnConstraints {
|
||||
interface DbtColumnConstraints {
|
||||
dbt: {
|
||||
not_null?: boolean;
|
||||
unique?: boolean;
|
||||
};
|
||||
}
|
||||
|
||||
export interface DbtParsedRelationship {
|
||||
interface DbtParsedRelationship {
|
||||
fromTable: string;
|
||||
fromColumn: string;
|
||||
toTable: string;
|
||||
|
|
@ -35,7 +35,7 @@ export interface DbtParsedRelationship {
|
|||
description?: string;
|
||||
}
|
||||
|
||||
export interface DbtParsedTable {
|
||||
interface DbtParsedTable {
|
||||
name: string;
|
||||
description: string | null;
|
||||
database: string | null;
|
||||
|
|
@ -126,6 +126,7 @@ type DbtSchemaDataTest =
|
|||
[key: string]: unknown;
|
||||
};
|
||||
|
||||
/** @internal */
|
||||
export function parseDbtSchemaFile(content: string, options: ParseDbtSchemaOptions = {}): DbtSchemaParseResult {
|
||||
return new DbtSchemaParser(options.logger ?? noopLogger).parseFile(content, options);
|
||||
}
|
||||
|
|
@ -138,13 +139,6 @@ export function parseDbtSchemaFiles(
|
|||
return new DbtSchemaParser(options.logger ?? noopLogger).parseFiles(files, variables, options.projectName ?? null);
|
||||
}
|
||||
|
||||
export function computeDbtSchemaHash(files: DbtSchemaFile[]): string {
|
||||
const combined = [...files]
|
||||
.sort((a, b) => a.path.localeCompare(b.path))
|
||||
.map((file) => `${file.path}:${file.content}`)
|
||||
.join('\n');
|
||||
return createHash('sha256').update(combined).digest('hex').substring(0, 16);
|
||||
}
|
||||
|
||||
class DbtSchemaParser {
|
||||
constructor(private readonly logger: KtxLogger) {}
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Add a link
Reference in a new issue