perf(setup): speed up conductor setup and make it rerun-safe (#107)

Drop the duplicate `pnpm run build` (artifacts:build already builds every
package). Run package builds in parallel topology via one recursive pnpm
invocation. Enable incremental tsc and keep the cli's tsbuildinfo outside
its dist (moved the dist wipe into a separate `clean` script). Run the
final `ktx status` doctor from a temp dir so it stops walking up into a
parent ktx.yaml and failing the script.

Conductor setup drops from ~26s to ~9.8s cold and ~4.4s warm.

scripts/conductor-scripts.test.mjs

@@ -27,9 +27,9 @@ describe('Conductor workspace scripts', () => {
     assert.match(setupScript, /uv sync --all-packages --all-groups/);
     assert.match(setupScript, /pnpm install --frozen-lockfile --prefer-offline/);
     assert.match(setupScript, /pnpm run native:rebuild/);
-    assert.match(setupScript, /pnpm run build/);
     assert.match(setupScript, /pnpm run artifacts:build/);
-    assert.match(setupScript, /packages\/cli\/dist\/bin\.js status --no-input/);
+    assert.match(setupScript, /packages\/cli\/dist\/bin\.js/);
+    assert.match(setupScript, /status --no-input/);
     assert.doesNotMatch(setupScript, /scripts\/conductor\//);
   });
 

scripts/conductor-setup.sh

@@ -133,13 +133,15 @@ pnpm install --frozen-lockfile --prefer-offline
 echo "Rebuilding native JS dependencies..."
 pnpm run native:rebuild
 
-echo "Building KTX packages..."
-pnpm run build
-
 echo "Building KTX runtime artifacts..."
+# Builds every internal package (llm/context/connectors/cli) before producing
+# the wheel + npm tarball, so a separate `pnpm run build` would be redundant.
 pnpm run artifacts:build
 
 echo "Running KTX setup doctor..."
-node packages/cli/dist/bin.js status --no-input
+# Run from a temp dir so `ktx status` doesn't walk up into a parent ktx.yaml
+# (e.g. ~/ktx.yaml) and report on an unrelated project.
+KTX_CLI_BIN="$PWD/packages/cli/dist/bin.js"
+( cd "${TMPDIR:-/tmp}" && node "$KTX_CLI_BIN" status --no-input )
 
 echo "=== Setup complete ==="

scripts/package-artifacts.mjs

@@ -41,12 +41,6 @@ export const NPM_ARTIFACT_PACKAGES = [{ name: PUBLIC_NPM_PACKAGE_NAME, packageRo
 
 export const CLI_PYTHON_ASSET_MANIFEST = 'manifest.json';
 
-const CONNECTOR_PACKAGE_NAMES = INTERNAL_NPM_WORKSPACE_PACKAGES
-  .map((packageInfo) => packageInfo.name)
-  .filter((packageName) => packageName.startsWith('@ktx/connector-'));
-
-const NPM_ARTIFACT_BUILD_ORDER = ['@ktx/llm', '@ktx/context', ...CONNECTOR_PACKAGE_NAMES, '@ktx/cli'];
-
 function scriptRootDir() {
   return resolve(dirname(fileURLToPath(import.meta.url)), '..');
 }
@@ -84,18 +78,19 @@ export function packageArtifactLayout(rootDir = scriptRootDir()) {
 }
 
 export function buildArtifactCommands(layout) {
-  const packagesByName = new Map(INTERNAL_NPM_WORKSPACE_PACKAGES.map((packageInfo) => [packageInfo.name, packageInfo]));
-  const npmBuildCommands = NPM_ARTIFACT_BUILD_ORDER.map((packageName) => {
-    const packageInfo = packagesByName.get(packageName);
-    if (!packageInfo) {
-      throw new Error(`Unknown npm artifact build package: ${packageName}`);
-    }
-    return {
-      command: 'pnpm',
-      args: ['--filter', packageInfo.name, 'run', 'build'],
-      cwd: layout.rootDir,
-    };
-  });
+  // One recursive pnpm invocation; topology comes from workspace deps in
+  // each package.json, parallelism from --workspace-concurrency.
+  const npmBuildCommand = {
+    command: 'pnpm',
+    args: [
+      '--filter',
+      './packages/*',
+      '--workspace-concurrency=10',
+      'run',
+      'build',
+    ],
+    cwd: layout.rootDir,
+  };
   const publicPackageCommand = {
     command: process.execPath,
     args: ['scripts/build-public-npm-package.mjs'],
@@ -103,7 +98,7 @@ export function buildArtifactCommands(layout) {
   };
 
   return [
-    ...npmBuildCommands,
+    npmBuildCommand,
     {
       command: process.execPath,
       args: ['scripts/build-python-runtime-wheel.mjs'],
@@ -929,21 +924,13 @@ async function buildArtifacts(layout) {
   await mkdir(layout.npmDir, { recursive: true });
   await mkdir(layout.pythonDir, { recursive: true });
 
-  const commands = buildArtifactCommands(layout);
-  const npmBuildCount = NPM_ARTIFACT_BUILD_ORDER.length;
-  const npmPackStart = commands.length - 1;
+  const [npmBuildCommand, wheelCommand, publicPackageCommand] = buildArtifactCommands(layout);
 
-  for (const command of commands.slice(0, npmBuildCount)) {
-    await runCommand(command.command, command.args, { cwd: command.cwd });
-  }
-  for (const command of commands.slice(npmBuildCount, npmPackStart)) {
-    await runCommand(command.command, command.args, { cwd: command.cwd });
-  }
+  await runCommand(npmBuildCommand.command, npmBuildCommand.args, { cwd: npmBuildCommand.cwd });
+  await runCommand(wheelCommand.command, wheelCommand.args, { cwd: wheelCommand.cwd });
   const pythonArtifacts = await findPythonArtifacts(layout.pythonDir);
   await copyRuntimeWheelAssets(layout, pythonArtifacts);
-  for (const command of commands.slice(npmPackStart)) {
-    await runCommand(command.command, command.args, { cwd: command.cwd });
-  }
+  await runCommand(publicPackageCommand.command, publicPackageCommand.args, { cwd: publicPackageCommand.cwd });
 
   for (const packageInfo of NPM_ARTIFACT_PACKAGES) {
     await assertPathExists(layout.npmTarballs[packageInfo.name], `${packageInfo.name} tarball`);

scripts/package-artifacts.test.mjs

@@ -31,12 +31,6 @@ async function writeJson(path, value) {
   await writeFile(path, `${JSON.stringify(value, null, 2)}\n`);
 }
 
-const INTERNAL_BUILD_PACKAGE_NAMES = INTERNAL_NPM_WORKSPACE_PACKAGES.map((packageInfo) => packageInfo.name);
-const CONNECTOR_PACKAGE_NAMES = INTERNAL_BUILD_PACKAGE_NAMES.filter((packageName) =>
-  packageName.startsWith('@ktx/connector-'),
-);
-const NPM_BUILD_PACKAGE_ORDER = ['@ktx/llm', '@ktx/context', ...CONNECTOR_PACKAGE_NAMES, '@ktx/cli'];
-
 async function writeReleaseMetadataInputs(root) {
   for (const packageInfo of INTERNAL_NPM_WORKSPACE_PACKAGES) {
     await mkdir(join(root, packageInfo.packageRoot), { recursive: true });
@@ -81,24 +75,17 @@ describe('packageArtifactLayout', () => {
 });
 
 describe('buildArtifactCommands', () => {
-  it('builds TypeScript packages and the runtime wheel before packing npm artifacts', () => {
+  it('builds TypeScript packages in parallel topology, then the runtime wheel, then packs npm artifacts', () => {
     const layout = packageArtifactLayout('/repo/ktx');
     const commands = buildArtifactCommands(layout);
 
     assert.deepEqual(
-      commands.slice(0, NPM_BUILD_PACKAGE_ORDER.length).map((command) => [command.command, command.args]),
-      NPM_BUILD_PACKAGE_ORDER.map((packageName) => ['pnpm', ['--filter', packageName, 'run', 'build']]),
-    );
-    assert.deepEqual(
-      commands.slice(NPM_BUILD_PACKAGE_ORDER.length, NPM_BUILD_PACKAGE_ORDER.length + 1).map((command) => [
-        command.command,
-        command.args,
-      ]),
-      [[process.execPath, ['scripts/build-python-runtime-wheel.mjs']]],
-    );
-    assert.deepEqual(
-      commands.slice(NPM_BUILD_PACKAGE_ORDER.length + 1).map((command) => [command.command, command.args]),
-      [[process.execPath, ['scripts/build-public-npm-package.mjs']]],
+      commands.map((command) => [command.command, command.args]),
+      [
+        ['pnpm', ['--filter', './packages/*', '--workspace-concurrency=10', 'run', 'build']],
+        [process.execPath, ['scripts/build-python-runtime-wheel.mjs']],
+        [process.execPath, ['scripts/build-public-npm-package.mjs']],
+      ],
     );
   });
 });