diff --git a/packages/cli/package.json b/packages/cli/package.json index 539618f7..2f1c4313 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -26,7 +26,8 @@ ], "scripts": { "assets:demo": "node scripts/build-demo-assets.mjs", - "build": "node -e \"fs.rmSync('dist', { recursive: true, force: true })\" && tsc -p tsconfig.json && node scripts/copy-runtime-assets.mjs && node ../../scripts/prepare-cli-bin.mjs", + "build": "tsc -p tsconfig.json && node scripts/copy-runtime-assets.mjs && node ../../scripts/prepare-cli-bin.mjs", + "clean": "node -e \"fs.rmSync('dist', { recursive: true, force: true }); fs.rmSync('node_modules/.cache/tsc.tsbuildinfo', { force: true })\"", "docs:commands": "pnpm run build && node dist/print-command-tree.js", "smoke": "vitest run src/standalone-smoke.test.ts src/example-smoke.test.ts --testTimeout 30000", "test": "vitest run --exclude src/standalone-smoke.test.ts --exclude src/example-smoke.test.ts --exclude src/setup-databases.test.ts --exclude src/scan.test.ts --exclude src/commands/connection-metabase-setup.test.ts --exclude src/setup-models.test.ts --exclude src/setup-sources.test.ts --exclude src/setup.test.ts --exclude src/connection.test.ts --exclude src/setup-embeddings.test.ts --exclude src/ingest.test.ts --exclude src/commands/connection-mapping.test.ts --exclude src/ingest-viz.test.ts --exclude src/demo.test.ts --exclude src/setup-project.test.ts --exclude src/sl.test.ts --exclude src/local-scan-connectors.test.ts --exclude src/commands/connection-notion.test.ts", diff --git a/packages/cli/tsconfig.json b/packages/cli/tsconfig.json index 40c250d6..b527c8a1 100644 --- a/packages/cli/tsconfig.json +++ b/packages/cli/tsconfig.json @@ -3,7 +3,8 @@ "compilerOptions": { "outDir": "./dist", "rootDir": "./src", - "jsx": "react-jsx" + "jsx": "react-jsx", + "tsBuildInfoFile": "./node_modules/.cache/tsc.tsbuildinfo" }, "include": ["src/**/*.ts", "src/**/*.tsx"], "exclude": ["dist", "node_modules"] diff --git a/scripts/conductor-scripts.test.mjs b/scripts/conductor-scripts.test.mjs index fa97a501..3284efae 100644 --- a/scripts/conductor-scripts.test.mjs +++ b/scripts/conductor-scripts.test.mjs @@ -27,9 +27,9 @@ describe('Conductor workspace scripts', () => { assert.match(setupScript, /uv sync --all-packages --all-groups/); assert.match(setupScript, /pnpm install --frozen-lockfile --prefer-offline/); assert.match(setupScript, /pnpm run native:rebuild/); - assert.match(setupScript, /pnpm run build/); assert.match(setupScript, /pnpm run artifacts:build/); - assert.match(setupScript, /packages\/cli\/dist\/bin\.js status --no-input/); + assert.match(setupScript, /packages\/cli\/dist\/bin\.js/); + assert.match(setupScript, /status --no-input/); assert.doesNotMatch(setupScript, /scripts\/conductor\//); }); diff --git a/scripts/conductor-setup.sh b/scripts/conductor-setup.sh index 6f270508..8d64e7ad 100755 --- a/scripts/conductor-setup.sh +++ b/scripts/conductor-setup.sh @@ -133,13 +133,15 @@ pnpm install --frozen-lockfile --prefer-offline echo "Rebuilding native JS dependencies..." pnpm run native:rebuild -echo "Building KTX packages..." -pnpm run build - echo "Building KTX runtime artifacts..." +# Builds every internal package (llm/context/connectors/cli) before producing +# the wheel + npm tarball, so a separate `pnpm run build` would be redundant. pnpm run artifacts:build echo "Running KTX setup doctor..." -node packages/cli/dist/bin.js status --no-input +# Run from a temp dir so `ktx status` doesn't walk up into a parent ktx.yaml +# (e.g. ~/ktx.yaml) and report on an unrelated project. +KTX_CLI_BIN="$PWD/packages/cli/dist/bin.js" +( cd "${TMPDIR:-/tmp}" && node "$KTX_CLI_BIN" status --no-input ) echo "=== Setup complete ===" diff --git a/scripts/package-artifacts.mjs b/scripts/package-artifacts.mjs index efe7f6f3..88654c55 100644 --- a/scripts/package-artifacts.mjs +++ b/scripts/package-artifacts.mjs @@ -41,12 +41,6 @@ export const NPM_ARTIFACT_PACKAGES = [{ name: PUBLIC_NPM_PACKAGE_NAME, packageRo export const CLI_PYTHON_ASSET_MANIFEST = 'manifest.json'; -const CONNECTOR_PACKAGE_NAMES = INTERNAL_NPM_WORKSPACE_PACKAGES - .map((packageInfo) => packageInfo.name) - .filter((packageName) => packageName.startsWith('@ktx/connector-')); - -const NPM_ARTIFACT_BUILD_ORDER = ['@ktx/llm', '@ktx/context', ...CONNECTOR_PACKAGE_NAMES, '@ktx/cli']; - function scriptRootDir() { return resolve(dirname(fileURLToPath(import.meta.url)), '..'); } @@ -84,18 +78,19 @@ export function packageArtifactLayout(rootDir = scriptRootDir()) { } export function buildArtifactCommands(layout) { - const packagesByName = new Map(INTERNAL_NPM_WORKSPACE_PACKAGES.map((packageInfo) => [packageInfo.name, packageInfo])); - const npmBuildCommands = NPM_ARTIFACT_BUILD_ORDER.map((packageName) => { - const packageInfo = packagesByName.get(packageName); - if (!packageInfo) { - throw new Error(`Unknown npm artifact build package: ${packageName}`); - } - return { - command: 'pnpm', - args: ['--filter', packageInfo.name, 'run', 'build'], - cwd: layout.rootDir, - }; - }); + // One recursive pnpm invocation; topology comes from workspace deps in + // each package.json, parallelism from --workspace-concurrency. + const npmBuildCommand = { + command: 'pnpm', + args: [ + '--filter', + './packages/*', + '--workspace-concurrency=10', + 'run', + 'build', + ], + cwd: layout.rootDir, + }; const publicPackageCommand = { command: process.execPath, args: ['scripts/build-public-npm-package.mjs'], @@ -103,7 +98,7 @@ export function buildArtifactCommands(layout) { }; return [ - ...npmBuildCommands, + npmBuildCommand, { command: process.execPath, args: ['scripts/build-python-runtime-wheel.mjs'], @@ -929,21 +924,13 @@ async function buildArtifacts(layout) { await mkdir(layout.npmDir, { recursive: true }); await mkdir(layout.pythonDir, { recursive: true }); - const commands = buildArtifactCommands(layout); - const npmBuildCount = NPM_ARTIFACT_BUILD_ORDER.length; - const npmPackStart = commands.length - 1; + const [npmBuildCommand, wheelCommand, publicPackageCommand] = buildArtifactCommands(layout); - for (const command of commands.slice(0, npmBuildCount)) { - await runCommand(command.command, command.args, { cwd: command.cwd }); - } - for (const command of commands.slice(npmBuildCount, npmPackStart)) { - await runCommand(command.command, command.args, { cwd: command.cwd }); - } + await runCommand(npmBuildCommand.command, npmBuildCommand.args, { cwd: npmBuildCommand.cwd }); + await runCommand(wheelCommand.command, wheelCommand.args, { cwd: wheelCommand.cwd }); const pythonArtifacts = await findPythonArtifacts(layout.pythonDir); await copyRuntimeWheelAssets(layout, pythonArtifacts); - for (const command of commands.slice(npmPackStart)) { - await runCommand(command.command, command.args, { cwd: command.cwd }); - } + await runCommand(publicPackageCommand.command, publicPackageCommand.args, { cwd: publicPackageCommand.cwd }); for (const packageInfo of NPM_ARTIFACT_PACKAGES) { await assertPathExists(layout.npmTarballs[packageInfo.name], `${packageInfo.name} tarball`); diff --git a/scripts/package-artifacts.test.mjs b/scripts/package-artifacts.test.mjs index 1f0bf164..7b0fc948 100644 --- a/scripts/package-artifacts.test.mjs +++ b/scripts/package-artifacts.test.mjs @@ -31,12 +31,6 @@ async function writeJson(path, value) { await writeFile(path, `${JSON.stringify(value, null, 2)}\n`); } -const INTERNAL_BUILD_PACKAGE_NAMES = INTERNAL_NPM_WORKSPACE_PACKAGES.map((packageInfo) => packageInfo.name); -const CONNECTOR_PACKAGE_NAMES = INTERNAL_BUILD_PACKAGE_NAMES.filter((packageName) => - packageName.startsWith('@ktx/connector-'), -); -const NPM_BUILD_PACKAGE_ORDER = ['@ktx/llm', '@ktx/context', ...CONNECTOR_PACKAGE_NAMES, '@ktx/cli']; - async function writeReleaseMetadataInputs(root) { for (const packageInfo of INTERNAL_NPM_WORKSPACE_PACKAGES) { await mkdir(join(root, packageInfo.packageRoot), { recursive: true }); @@ -81,24 +75,17 @@ describe('packageArtifactLayout', () => { }); describe('buildArtifactCommands', () => { - it('builds TypeScript packages and the runtime wheel before packing npm artifacts', () => { + it('builds TypeScript packages in parallel topology, then the runtime wheel, then packs npm artifacts', () => { const layout = packageArtifactLayout('/repo/ktx'); const commands = buildArtifactCommands(layout); assert.deepEqual( - commands.slice(0, NPM_BUILD_PACKAGE_ORDER.length).map((command) => [command.command, command.args]), - NPM_BUILD_PACKAGE_ORDER.map((packageName) => ['pnpm', ['--filter', packageName, 'run', 'build']]), - ); - assert.deepEqual( - commands.slice(NPM_BUILD_PACKAGE_ORDER.length, NPM_BUILD_PACKAGE_ORDER.length + 1).map((command) => [ - command.command, - command.args, - ]), - [[process.execPath, ['scripts/build-python-runtime-wheel.mjs']]], - ); - assert.deepEqual( - commands.slice(NPM_BUILD_PACKAGE_ORDER.length + 1).map((command) => [command.command, command.args]), - [[process.execPath, ['scripts/build-public-npm-package.mjs']]], + commands.map((command) => [command.command, command.args]), + [ + ['pnpm', ['--filter', './packages/*', '--workspace-concurrency=10', 'run', 'build']], + [process.execPath, ['scripts/build-python-runtime-wheel.mjs']], + [process.execPath, ['scripts/build-public-npm-package.mjs']], + ], ); }); }); diff --git a/tsconfig.base.json b/tsconfig.base.json index 1976d77e..df93e255 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -6,6 +6,7 @@ "target": "ES2023", "lib": ["ES2023"], "declaration": true, + "incremental": true, "strict": true, "strictNullChecks": true, "esModuleInterop": true,