From 2de4dd2c1b7a2f1da785fc5e0ff8e23bf3a7b486 Mon Sep 17 00:00:00 2001 From: Andrey Avtomonov Date: Fri, 15 May 2026 12:06:37 +0200 Subject: [PATCH] perf(setup): speed up conductor setup and make it rerun-safe (#107) Drop the duplicate `pnpm run build` (artifacts:build already builds every package). Run package builds in parallel topology via one recursive pnpm invocation. Enable incremental tsc and keep the cli's tsbuildinfo outside its dist (moved the dist wipe into a separate `clean` script). Run the final `ktx status` doctor from a temp dir so it stops walking up into a parent ktx.yaml and failing the script. Conductor setup drops from ~26s to ~9.8s cold and ~4.4s warm. --- packages/cli/package.json | 3 +- packages/cli/tsconfig.json | 3 +- scripts/conductor-scripts.test.mjs | 4 +-- scripts/conductor-setup.sh | 10 +++--- scripts/package-artifacts.mjs | 49 +++++++++++------------------- scripts/package-artifacts.test.mjs | 27 +++++----------- tsconfig.base.json | 1 + 7 files changed, 38 insertions(+), 59 deletions(-) diff --git a/packages/cli/package.json b/packages/cli/package.json index 539618f7..2f1c4313 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -26,7 +26,8 @@ ], "scripts": { "assets:demo": "node scripts/build-demo-assets.mjs", - "build": "node -e \"fs.rmSync('dist', { recursive: true, force: true })\" && tsc -p tsconfig.json && node scripts/copy-runtime-assets.mjs && node ../../scripts/prepare-cli-bin.mjs", + "build": "tsc -p tsconfig.json && node scripts/copy-runtime-assets.mjs && node ../../scripts/prepare-cli-bin.mjs", + "clean": "node -e \"fs.rmSync('dist', { recursive: true, force: true }); fs.rmSync('node_modules/.cache/tsc.tsbuildinfo', { force: true })\"", "docs:commands": "pnpm run build && node dist/print-command-tree.js", "smoke": "vitest run src/standalone-smoke.test.ts src/example-smoke.test.ts --testTimeout 30000", "test": "vitest run --exclude src/standalone-smoke.test.ts --exclude src/example-smoke.test.ts --exclude src/setup-databases.test.ts --exclude src/scan.test.ts --exclude src/commands/connection-metabase-setup.test.ts --exclude src/setup-models.test.ts --exclude src/setup-sources.test.ts --exclude src/setup.test.ts --exclude src/connection.test.ts --exclude src/setup-embeddings.test.ts --exclude src/ingest.test.ts --exclude src/commands/connection-mapping.test.ts --exclude src/ingest-viz.test.ts --exclude src/demo.test.ts --exclude src/setup-project.test.ts --exclude src/sl.test.ts --exclude src/local-scan-connectors.test.ts --exclude src/commands/connection-notion.test.ts", diff --git a/packages/cli/tsconfig.json b/packages/cli/tsconfig.json index 40c250d6..b527c8a1 100644 --- a/packages/cli/tsconfig.json +++ b/packages/cli/tsconfig.json @@ -3,7 +3,8 @@ "compilerOptions": { "outDir": "./dist", "rootDir": "./src", - "jsx": "react-jsx" + "jsx": "react-jsx", + "tsBuildInfoFile": "./node_modules/.cache/tsc.tsbuildinfo" }, "include": ["src/**/*.ts", "src/**/*.tsx"], "exclude": ["dist", "node_modules"] diff --git a/scripts/conductor-scripts.test.mjs b/scripts/conductor-scripts.test.mjs index fa97a501..3284efae 100644 --- a/scripts/conductor-scripts.test.mjs +++ b/scripts/conductor-scripts.test.mjs @@ -27,9 +27,9 @@ describe('Conductor workspace scripts', () => { assert.match(setupScript, /uv sync --all-packages --all-groups/); assert.match(setupScript, /pnpm install --frozen-lockfile --prefer-offline/); assert.match(setupScript, /pnpm run native:rebuild/); - assert.match(setupScript, /pnpm run build/); assert.match(setupScript, /pnpm run artifacts:build/); - assert.match(setupScript, /packages\/cli\/dist\/bin\.js status --no-input/); + assert.match(setupScript, /packages\/cli\/dist\/bin\.js/); + assert.match(setupScript, /status --no-input/); assert.doesNotMatch(setupScript, /scripts\/conductor\//); }); diff --git a/scripts/conductor-setup.sh b/scripts/conductor-setup.sh index 6f270508..8d64e7ad 100755 --- a/scripts/conductor-setup.sh +++ b/scripts/conductor-setup.sh @@ -133,13 +133,15 @@ pnpm install --frozen-lockfile --prefer-offline echo "Rebuilding native JS dependencies..." pnpm run native:rebuild -echo "Building KTX packages..." -pnpm run build - echo "Building KTX runtime artifacts..." +# Builds every internal package (llm/context/connectors/cli) before producing +# the wheel + npm tarball, so a separate `pnpm run build` would be redundant. pnpm run artifacts:build echo "Running KTX setup doctor..." -node packages/cli/dist/bin.js status --no-input +# Run from a temp dir so `ktx status` doesn't walk up into a parent ktx.yaml +# (e.g. ~/ktx.yaml) and report on an unrelated project. +KTX_CLI_BIN="$PWD/packages/cli/dist/bin.js" +( cd "${TMPDIR:-/tmp}" && node "$KTX_CLI_BIN" status --no-input ) echo "=== Setup complete ===" diff --git a/scripts/package-artifacts.mjs b/scripts/package-artifacts.mjs index efe7f6f3..88654c55 100644 --- a/scripts/package-artifacts.mjs +++ b/scripts/package-artifacts.mjs @@ -41,12 +41,6 @@ export const NPM_ARTIFACT_PACKAGES = [{ name: PUBLIC_NPM_PACKAGE_NAME, packageRo export const CLI_PYTHON_ASSET_MANIFEST = 'manifest.json'; -const CONNECTOR_PACKAGE_NAMES = INTERNAL_NPM_WORKSPACE_PACKAGES - .map((packageInfo) => packageInfo.name) - .filter((packageName) => packageName.startsWith('@ktx/connector-')); - -const NPM_ARTIFACT_BUILD_ORDER = ['@ktx/llm', '@ktx/context', ...CONNECTOR_PACKAGE_NAMES, '@ktx/cli']; - function scriptRootDir() { return resolve(dirname(fileURLToPath(import.meta.url)), '..'); } @@ -84,18 +78,19 @@ export function packageArtifactLayout(rootDir = scriptRootDir()) { } export function buildArtifactCommands(layout) { - const packagesByName = new Map(INTERNAL_NPM_WORKSPACE_PACKAGES.map((packageInfo) => [packageInfo.name, packageInfo])); - const npmBuildCommands = NPM_ARTIFACT_BUILD_ORDER.map((packageName) => { - const packageInfo = packagesByName.get(packageName); - if (!packageInfo) { - throw new Error(`Unknown npm artifact build package: ${packageName}`); - } - return { - command: 'pnpm', - args: ['--filter', packageInfo.name, 'run', 'build'], - cwd: layout.rootDir, - }; - }); + // One recursive pnpm invocation; topology comes from workspace deps in + // each package.json, parallelism from --workspace-concurrency. + const npmBuildCommand = { + command: 'pnpm', + args: [ + '--filter', + './packages/*', + '--workspace-concurrency=10', + 'run', + 'build', + ], + cwd: layout.rootDir, + }; const publicPackageCommand = { command: process.execPath, args: ['scripts/build-public-npm-package.mjs'], @@ -103,7 +98,7 @@ export function buildArtifactCommands(layout) { }; return [ - ...npmBuildCommands, + npmBuildCommand, { command: process.execPath, args: ['scripts/build-python-runtime-wheel.mjs'], @@ -929,21 +924,13 @@ async function buildArtifacts(layout) { await mkdir(layout.npmDir, { recursive: true }); await mkdir(layout.pythonDir, { recursive: true }); - const commands = buildArtifactCommands(layout); - const npmBuildCount = NPM_ARTIFACT_BUILD_ORDER.length; - const npmPackStart = commands.length - 1; + const [npmBuildCommand, wheelCommand, publicPackageCommand] = buildArtifactCommands(layout); - for (const command of commands.slice(0, npmBuildCount)) { - await runCommand(command.command, command.args, { cwd: command.cwd }); - } - for (const command of commands.slice(npmBuildCount, npmPackStart)) { - await runCommand(command.command, command.args, { cwd: command.cwd }); - } + await runCommand(npmBuildCommand.command, npmBuildCommand.args, { cwd: npmBuildCommand.cwd }); + await runCommand(wheelCommand.command, wheelCommand.args, { cwd: wheelCommand.cwd }); const pythonArtifacts = await findPythonArtifacts(layout.pythonDir); await copyRuntimeWheelAssets(layout, pythonArtifacts); - for (const command of commands.slice(npmPackStart)) { - await runCommand(command.command, command.args, { cwd: command.cwd }); - } + await runCommand(publicPackageCommand.command, publicPackageCommand.args, { cwd: publicPackageCommand.cwd }); for (const packageInfo of NPM_ARTIFACT_PACKAGES) { await assertPathExists(layout.npmTarballs[packageInfo.name], `${packageInfo.name} tarball`); diff --git a/scripts/package-artifacts.test.mjs b/scripts/package-artifacts.test.mjs index 1f0bf164..7b0fc948 100644 --- a/scripts/package-artifacts.test.mjs +++ b/scripts/package-artifacts.test.mjs @@ -31,12 +31,6 @@ async function writeJson(path, value) { await writeFile(path, `${JSON.stringify(value, null, 2)}\n`); } -const INTERNAL_BUILD_PACKAGE_NAMES = INTERNAL_NPM_WORKSPACE_PACKAGES.map((packageInfo) => packageInfo.name); -const CONNECTOR_PACKAGE_NAMES = INTERNAL_BUILD_PACKAGE_NAMES.filter((packageName) => - packageName.startsWith('@ktx/connector-'), -); -const NPM_BUILD_PACKAGE_ORDER = ['@ktx/llm', '@ktx/context', ...CONNECTOR_PACKAGE_NAMES, '@ktx/cli']; - async function writeReleaseMetadataInputs(root) { for (const packageInfo of INTERNAL_NPM_WORKSPACE_PACKAGES) { await mkdir(join(root, packageInfo.packageRoot), { recursive: true }); @@ -81,24 +75,17 @@ describe('packageArtifactLayout', () => { }); describe('buildArtifactCommands', () => { - it('builds TypeScript packages and the runtime wheel before packing npm artifacts', () => { + it('builds TypeScript packages in parallel topology, then the runtime wheel, then packs npm artifacts', () => { const layout = packageArtifactLayout('/repo/ktx'); const commands = buildArtifactCommands(layout); assert.deepEqual( - commands.slice(0, NPM_BUILD_PACKAGE_ORDER.length).map((command) => [command.command, command.args]), - NPM_BUILD_PACKAGE_ORDER.map((packageName) => ['pnpm', ['--filter', packageName, 'run', 'build']]), - ); - assert.deepEqual( - commands.slice(NPM_BUILD_PACKAGE_ORDER.length, NPM_BUILD_PACKAGE_ORDER.length + 1).map((command) => [ - command.command, - command.args, - ]), - [[process.execPath, ['scripts/build-python-runtime-wheel.mjs']]], - ); - assert.deepEqual( - commands.slice(NPM_BUILD_PACKAGE_ORDER.length + 1).map((command) => [command.command, command.args]), - [[process.execPath, ['scripts/build-public-npm-package.mjs']]], + commands.map((command) => [command.command, command.args]), + [ + ['pnpm', ['--filter', './packages/*', '--workspace-concurrency=10', 'run', 'build']], + [process.execPath, ['scripts/build-python-runtime-wheel.mjs']], + [process.execPath, ['scripts/build-public-npm-package.mjs']], + ], ); }); }); diff --git a/tsconfig.base.json b/tsconfig.base.json index 1976d77e..df93e255 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -6,6 +6,7 @@ "target": "ES2023", "lib": ["ES2023"], "declaration": true, + "incremental": true, "strict": true, "strictNullChecks": true, "esModuleInterop": true,