Merge origin/main into npx-ktx-python-daemon

scripts/build-benchmark-snapshot.test.mjs

@@ -1,4 +1,5 @@
 import assert from 'node:assert/strict';
+import { readFile } from 'node:fs/promises';
 import { createRequire } from 'node:module';
 import { describe, it } from 'node:test';
 import { buildBenchmarkSnapshot } from './build-benchmark-snapshot.mjs';
@@ -250,4 +251,13 @@ describe('buildBenchmarkSnapshot', () => {
       },
     ]);
   });
+
+  it('exposes relationship benchmarks as an explicit context package script', async () => {
+    const packageJson = JSON.parse(await readFile(new URL('../packages/context/package.json', import.meta.url), 'utf8'));
+
+    assert.equal(
+      packageJson.scripts['relationships:benchmarks:test'],
+      'KTX_RUN_RELATIONSHIP_BENCHMARKS=1 vitest run src/scan/relationship-benchmarks.test.ts',
+    );
+  });
 });

scripts/check-boundaries.test.mjs

@@ -14,7 +14,7 @@ function lowerProductName() {
 describe('scanFileContent', () => {
   it('rejects source imports from application directories', () => {
     const serverAlias = '@' + 'server/contracts';
-    const pythonAppPath = 'python-service/' + 'app/api/endpoints/semantic_layer.py';
+    const pythonAppPath = `${['python', 'service'].join('-')}/app/api/endpoints/semantic_layer.py`;
 
     const violations = [
       ...scanFileContent('packages/context/src/index.ts', `import { orpc } from '${serverAlias}';`),

scripts/ci-artifact-upload.test.mjs

@@ -4,7 +4,7 @@ import { dirname, resolve } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { describe, it } from 'node:test';
 
-const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..', '..');
+const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..');
 const ciWorkflowPath = resolve(repoRoot, '.github', 'workflows', 'ci.yml');
 
 async function readCiWorkflowOrSkip(testContext) {
@@ -21,7 +21,7 @@ async function readCiWorkflowOrSkip(testContext) {
 }
 
 describe('KTX CI artifact upload contract', () => {
-  it('uploads verified KTX package artifacts from check-ktx-subtree', async (testContext) => {
+  it('uploads verified KTX package artifacts from the standalone check job', async (testContext) => {
     const workflow = await readCiWorkflowOrSkip(testContext);
     if (workflow === null) {
       return;
@@ -29,42 +29,35 @@ describe('KTX CI artifact upload contract', () => {
 
     assert.match(
       workflow,
-      /name: Build ktx package artifacts and verify public smoke\s+run: cd ktx && pnpm run artifacts:build && pnpm run artifacts:verify-manifest && pnpm run artifacts:verify-demo\s+- name: Upload ktx package artifacts/s,
+      /name: Build and verify package artifacts\s+run: pnpm run artifacts:check\s+- name: Upload package artifacts/s,
     );
     assert.match(workflow, /uses: actions\/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f/);
     assert.match(workflow, /name: ktx-package-artifacts-\$\{\{ github\.sha \}\}/);
-    assert.match(workflow, /ktx\/dist\/artifacts\/manifest\.json/);
-    assert.match(workflow, /ktx\/dist\/artifacts\/npm\/\*\.tgz/);
-    assert.match(workflow, /ktx\/dist\/artifacts\/python\/\*\.whl/);
-    assert.match(workflow, /ktx\/dist\/artifacts\/python\/\*\.tar\.gz/);
+    assert.match(workflow, /dist\/artifacts\/manifest\.json/);
+    assert.match(workflow, /dist\/artifacts\/npm\/\*\.tgz/);
+    assert.match(workflow, /dist\/artifacts\/python\/\*\.whl/);
+    assert.match(workflow, /dist\/artifacts\/python\/\*\.tar\.gz/);
     assert.match(workflow, /if-no-files-found: error/);
     assert.match(workflow, /retention-days: 7/);
   });
 
-  it('runs packed demo artifact smoke on Linux and macOS', async (testContext) => {
+  it('runs TypeScript and Python checks in the standalone workflow', async (testContext) => {
     const workflow = await readCiWorkflowOrSkip(testContext);
     if (workflow === null) {
       return;
     }
 
-    assert.match(workflow, /check-ktx-packed-demo:/);
-    assert.match(workflow, /matrix:\s+os: \[ubuntu-latest, macos-latest\]/s);
-    assert.match(workflow, /name: Download ktx package artifacts/);
-    assert.match(workflow, /path: ktx\/dist\/artifacts/);
-    assert.match(workflow, /run: cd ktx && pnpm run artifacts:verify-demo/);
+    assert.match(workflow, /run: pnpm run check/);
+    assert.match(workflow, /run: uv sync --all-packages/);
+    assert.match(workflow, /run: uv run pytest/);
   });
 
-  it('includes packed demo artifact smoke in ci-success', async (testContext) => {
+  it('does not depend on host application CI jobs', async (testContext) => {
     const workflow = await readCiWorkflowOrSkip(testContext);
     if (workflow === null) {
       return;
     }
 
-    assert.match(
-      workflow,
-      /needs: \[check-ktx-subtree, check-ktx-packed-demo, build-python-service, test-server, build-frontend, run-pre-commit, build-docker-images\]/,
-    );
-    assert.match(workflow, /needs\.check-ktx-packed-demo\.result.*== "failure"/);
-    assert.match(workflow, /needs\.check-ktx-packed-demo\.result.*== "cancelled"/);
+    assert.doesNotMatch(workflow, /build-python-service|test-server|build-frontend|build-docker-images/);
   });
 });

scripts/conductor-run.sh

@@ -0,0 +1,98 @@
+#!/bin/bash
+# conductor-run.sh - Starts the long-lived local KTX daemon for Conductor.
+#
+# Uses a fixed port because Conductor runs this workspace in nonconcurrent mode.
+
+set -e
+set -o pipefail
+
+read_required_uv_version() {
+  local project_file="$1"
+
+  if [ ! -f "$project_file" ]; then
+    return 1
+  fi
+
+  sed -nE 's/^[[:space:]]*required-version[[:space:]]*=[[:space:]]*"([^"]+)".*/\1/p' "$project_file" | head -n 1
+}
+
+uv_version() {
+  local uv_bin="$1"
+
+  "$uv_bin" --version 2>/dev/null | awk '{print $2}'
+}
+
+install_workspace_uv() {
+  local required_version="$1"
+  local install_dir="$PWD/.context/bin/uv-$required_version"
+
+  mkdir -p "$install_dir"
+
+  if [ ! -x "$install_dir/uv" ] || [ "$(uv_version "$install_dir/uv")" != "$required_version" ]; then
+    echo "Installing workspace-local uv $required_version..." >&2
+    curl -LsSf "https://astral.sh/uv/$required_version/install.sh" |
+      env UV_INSTALL_DIR="$install_dir" UV_NO_MODIFY_PATH=1 sh >&2
+  fi
+
+  printf '%s\n' "$install_dir/uv"
+}
+
+resolve_uv_for_project() {
+  local project_file="$1"
+  local required_version
+  local system_uv
+  local system_version
+  local workspace_uv
+
+  required_version="$(read_required_uv_version "$project_file" || true)"
+  required_version="${required_version#==}"
+
+  if [ -z "$required_version" ]; then
+    command -v uv
+    return
+  fi
+
+  if ! [[ "$required_version" =~ ^[0-9]+[.][0-9]+[.][0-9]+$ ]]; then
+    echo "WARNING: Unsupported uv required-version '$required_version'; using uv from PATH." >&2
+    command -v uv
+    return
+  fi
+
+  if command -v uv >/dev/null 2>&1; then
+    system_uv="$(command -v uv)"
+    system_version="$(uv_version "$system_uv")"
+
+    if [ "$system_version" = "$required_version" ]; then
+      printf '%s\n' "$system_uv"
+      return
+    fi
+
+    echo "Found uv $system_version at $system_uv; $project_file requires uv $required_version." >&2
+  else
+    echo "uv is not installed on PATH; $project_file requires uv $required_version." >&2
+  fi
+
+  workspace_uv="$(install_workspace_uv "$required_version")"
+
+  if [ "$(uv_version "$workspace_uv")" != "$required_version" ]; then
+    echo "ERROR: Expected uv $required_version at $workspace_uv, got $("$workspace_uv" --version 2>&1 || true)." >&2
+    return 1
+  fi
+
+  printf '%s\n' "$workspace_uv"
+}
+
+echo "=== Starting KTX for Conductor ==="
+
+echo "Building KTX packages..."
+pnpm run build
+
+KTX_UV_BIN="$(resolve_uv_for_project "pyproject.toml")"
+export PATH="$(dirname "$KTX_UV_BIN"):$PATH"
+
+if [ -f ".venv/bin/activate" ]; then
+  source .venv/bin/activate
+fi
+
+echo "KTX daemon: http://127.0.0.1:8765"
+exec uv run ktx-daemon serve-http --host 127.0.0.1 --port 8765

scripts/conductor-scripts.test.mjs

@@ -0,0 +1,50 @@
+import assert from 'node:assert/strict';
+import { readFile } from 'node:fs/promises';
+import { describe, it } from 'node:test';
+
+async function readText(relativePath) {
+  return readFile(new URL(`../${relativePath}`, import.meta.url), 'utf8');
+}
+
+describe('Conductor workspace scripts', () => {
+  it('registers setup and run scripts in nonconcurrent mode', async () => {
+    const manifest = JSON.parse(await readText('conductor.json'));
+
+    assert.deepEqual(manifest.scripts, {
+      setup: 'bash scripts/conductor-setup.sh',
+      run: 'bash scripts/conductor-run.sh',
+    });
+    assert.equal(manifest.runScriptMode, 'nonconcurrent');
+  });
+
+  it('sets up exact uv, local files, Python packages, JS packages, and the built CLI', async () => {
+    const setupScript = await readText('scripts/conductor-setup.sh');
+
+    assert.match(setupScript, /read_required_uv_version\(\)/);
+    assert.match(setupScript, /\.context\/bin\/uv-\$required_version/);
+    assert.match(setupScript, /link_agent_overlays/);
+    assert.match(setupScript, /CONDUCTOR_ROOT_PATH/);
+    assert.match(setupScript, /uv sync --all-packages --all-groups/);
+    assert.match(setupScript, /pnpm install --frozen-lockfile --prefer-offline/);
+    assert.match(setupScript, /pnpm run native:rebuild/);
+    assert.match(setupScript, /pnpm run build/);
+    assert.match(setupScript, /packages\/cli\/dist\/bin\.js dev doctor setup --no-input/);
+    assert.doesNotMatch(setupScript, /scripts\/conductor\//);
+  });
+
+  it('links private agent overlays when KTX_AGENT_OVERLAYS_ROOT is set', async () => {
+    const workspaceScript = await readText('scripts/conductor-setup.sh');
+
+    assert.match(workspaceScript, /KTX_AGENT_OVERLAYS_ROOT/);
+    assert.match(workspaceScript, /ln -s "\$\{KTX_AGENT_OVERLAYS_ROOT\}\/\.agents" \.agents/);
+  });
+
+  it('runs the KTX daemon on the documented fixed local port', async () => {
+    const runScript = await readText('scripts/conductor-run.sh');
+
+    assert.match(runScript, /pnpm run build/);
+    assert.match(runScript, /source \.venv\/bin\/activate/);
+    assert.match(runScript, /uv run ktx-daemon serve-http --host 127\.0\.0\.1 --port 8765/);
+    assert.doesNotMatch(runScript, /\bnpx\b/);
+  });
+});

scripts/conductor-setup.sh

@@ -0,0 +1,129 @@
+#!/bin/bash
+# conductor-setup.sh - Runs once when Conductor creates a KTX workspace.
+#
+# Prepares the standalone pnpm + uv workspace and builds the local CLI.
+
+set -e
+set -o pipefail
+
+read_required_uv_version() {
+  local project_file="$1"
+
+  if [ ! -f "$project_file" ]; then
+    return 1
+  fi
+
+  sed -nE 's/^[[:space:]]*required-version[[:space:]]*=[[:space:]]*"([^"]+)".*/\1/p' "$project_file" | head -n 1
+}
+
+uv_version() {
+  local uv_bin="$1"
+
+  "$uv_bin" --version 2>/dev/null | awk '{print $2}'
+}
+
+install_workspace_uv() {
+  local required_version="$1"
+  local install_dir="$PWD/.context/bin/uv-$required_version"
+
+  mkdir -p "$install_dir"
+
+  if [ ! -x "$install_dir/uv" ] || [ "$(uv_version "$install_dir/uv")" != "$required_version" ]; then
+    echo "Installing workspace-local uv $required_version..." >&2
+    curl -LsSf "https://astral.sh/uv/$required_version/install.sh" |
+      env UV_INSTALL_DIR="$install_dir" UV_NO_MODIFY_PATH=1 sh >&2
+  fi
+
+  printf '%s\n' "$install_dir/uv"
+}
+
+resolve_uv_for_project() {
+  local project_file="$1"
+  local required_version
+  local system_uv
+  local system_version
+  local workspace_uv
+
+  required_version="$(read_required_uv_version "$project_file" || true)"
+  required_version="${required_version#==}"
+
+  if [ -z "$required_version" ]; then
+    command -v uv
+    return
+  fi
+
+  if ! [[ "$required_version" =~ ^[0-9]+[.][0-9]+[.][0-9]+$ ]]; then
+    echo "WARNING: Unsupported uv required-version '$required_version'; using uv from PATH." >&2
+    command -v uv
+    return
+  fi
+
+  if command -v uv >/dev/null 2>&1; then
+    system_uv="$(command -v uv)"
+    system_version="$(uv_version "$system_uv")"
+
+    if [ "$system_version" = "$required_version" ]; then
+      printf '%s\n' "$system_uv"
+      return
+    fi
+
+    echo "Found uv $system_version at $system_uv; $project_file requires uv $required_version." >&2
+  else
+    echo "uv is not installed on PATH; $project_file requires uv $required_version." >&2
+  fi
+
+  workspace_uv="$(install_workspace_uv "$required_version")"
+
+  if [ "$(uv_version "$workspace_uv")" != "$required_version" ]; then
+    echo "ERROR: Expected uv $required_version at $workspace_uv, got $("$workspace_uv" --version 2>&1 || true)." >&2
+    return 1
+  fi
+
+  printf '%s\n' "$workspace_uv"
+}
+
+link_agent_overlays() {
+  if [ -z "${KTX_AGENT_OVERLAYS_ROOT:-}" ] || [ ! -d "${KTX_AGENT_OVERLAYS_ROOT}/.agents" ]; then
+    return 0
+  fi
+
+  if [ -L .agents ]; then
+    return 0
+  fi
+
+  if [ -e .agents ]; then
+    echo "Skipping .agents symlink because .agents already exists and is not a symlink." >&2
+    return 0
+  fi
+
+  ln -s "${KTX_AGENT_OVERLAYS_ROOT}/.agents" .agents
+}
+
+echo "=== Conductor KTX workspace setup ==="
+
+link_agent_overlays
+
+if [ -n "${CONDUCTOR_ROOT_PATH:-}" ] && [ -f "$CONDUCTOR_ROOT_PATH/.env" ]; then
+  ln -sf "$CONDUCTOR_ROOT_PATH/.env" .env
+  echo "Linked .env"
+fi
+
+KTX_UV_BIN="$(resolve_uv_for_project "pyproject.toml")"
+export PATH="$(dirname "$KTX_UV_BIN"):$PATH"
+
+echo "Installing KTX Python dependencies..."
+uv sync --all-packages --all-groups
+
+echo "Installing KTX JS dependencies..."
+pnpm install --frozen-lockfile --prefer-offline
+
+echo "Rebuilding native JS dependencies..."
+pnpm run native:rebuild
+
+echo "Building KTX packages..."
+pnpm run build
+
+echo "Running KTX setup doctor..."
+node packages/cli/dist/bin.js dev doctor setup --no-input
+
+echo "=== Setup complete ==="

scripts/examples-docs.test.mjs

@@ -88,6 +88,8 @@ describe('standalone example docs', () => {
     assert.match(smoke, /assert_manifest "\$FIRST_MANIFEST" true/);
     assert.match(smoke, /assert_manifest "\$SECOND_MANIFEST" false/);
     assert.match(smoke, /assert_manifest "\$RESET_MANIFEST" true/);
+    assert.doesNotMatch(readme, /python-service/);
+    assert.doesNotMatch(smoke, /python-service|PYTHON_SERVICE|REPO_ROOT/);
   });
 
   it('lists every published TypeScript package in the package root README', async () => {
@@ -99,7 +101,6 @@ describe('standalone example docs', () => {
     assert.match(rootReadme, /`packages\/connector-clickhouse`/);
     assert.match(rootReadme, /`packages\/connector-mysql`/);
     assert.match(rootReadme, /`packages\/connector-postgres`/);
-    assert.match(rootReadme, /`packages\/connector-posthog`/);
     assert.match(rootReadme, /`packages\/connector-snowflake`/);
     assert.match(rootReadme, /`packages\/connector-sqlite`/);
     assert.match(rootReadme, /`packages\/connector-sqlserver`/);
@@ -205,6 +206,8 @@ describe('standalone example docs', () => {
 
     assert.match(packageJson.scripts.smoke, /src\/standalone-smoke\.test\.ts/);
     assert.match(packageJson.scripts.smoke, /src\/example-smoke\.test\.ts/);
+    assert.match(packageJson.scripts.test, /--exclude src\/standalone-smoke\.test\.ts/);
+    assert.match(packageJson.scripts.test, /--exclude src\/example-smoke\.test\.ts/);
   });
 
   it('documents daemon HTTP database, source generation, LookML, embedding, and code execution support', async () => {

scripts/package-artifacts.mjs

@@ -31,7 +31,6 @@ export const INTERNAL_NPM_WORKSPACE_PACKAGES = [
   { name: '@ktx/connector-clickhouse', packageRoot: 'packages/connector-clickhouse' },
   { name: '@ktx/connector-mysql', packageRoot: 'packages/connector-mysql' },
   { name: '@ktx/connector-postgres', packageRoot: 'packages/connector-postgres' },
-  { name: '@ktx/connector-posthog', packageRoot: 'packages/connector-posthog' },
   { name: '@ktx/connector-snowflake', packageRoot: 'packages/connector-snowflake' },
   { name: '@ktx/connector-sqlite', packageRoot: 'packages/connector-sqlite' },
   { name: '@ktx/connector-sqlserver', packageRoot: 'packages/connector-sqlserver' },

scripts/precommit-check.mjs

@@ -15,7 +15,6 @@ const packageNameByDir = new Map(
     'connector-clickhouse',
     'connector-mysql',
     'connector-postgres',
-    'connector-posthog',
     'connector-snowflake',
     'connector-sqlite',
     'connector-sqlserver',

scripts/precommit-check.test.mjs

@@ -9,7 +9,7 @@ function commandKeys(files) {
 
 describe('precommit-check', () => {
   it('skips files outside ktx', () => {
-    assert.deepEqual(commandKeys(['server/src/app.ts']), []);
+    assert.deepEqual(commandKeys(['outside-workspace/src/app.ts']), []);
   });
 
   it('runs only the touched package checks for package code', () => {

scripts/standalone-ci-workflow.test.mjs

@@ -27,6 +27,8 @@ describe('standalone KTX CI workflow', () => {
       'cache-dependency-path: "pnpm-lock.yaml"',
       'pnpm install --frozen-lockfile',
       'pnpm run check',
+      'pnpm run test:slow',
+      'pnpm run smoke',
       'actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405',
       'python-version: "3.13"',
       'astral-sh/setup-uv@eac588ad8def6316056a12d4907a9d4d84ff7a3b',

scripts/test-tiering.test.mjs

@@ -0,0 +1,73 @@
+import assert from 'node:assert/strict';
+import { readFile } from 'node:fs/promises';
+import { describe, it } from 'node:test';
+
+async function readJson(path) {
+  return JSON.parse(await readFile(new URL(path, import.meta.url), 'utf8'));
+}
+
+function assertScriptContainsAll(script, expected) {
+  for (const item of expected) {
+    assert.match(script, new RegExp(item.replaceAll('/', '\\/').replaceAll('.', '\\.')));
+  }
+}
+
+describe('test tiering', () => {
+  const cliSlowTests = [
+    'src/setup-databases.test.ts',
+    'src/scan.test.ts',
+    'src/commands/connection-metabase-setup.test.ts',
+    'src/setup-models.test.ts',
+    'src/setup-sources.test.ts',
+    'src/setup.test.ts',
+    'src/connection.test.ts',
+    'src/setup-embeddings.test.ts',
+    'src/ingest.test.ts',
+    'src/commands/connection-mapping.test.ts',
+    'src/ingest-viz.test.ts',
+    'src/demo.test.ts',
+    'src/setup-project.test.ts',
+    'src/sl.test.ts',
+    'src/local-scan-connectors.test.ts',
+    'src/commands/connection-notion.test.ts',
+  ];
+
+  const contextSlowTests = [
+    'src/scan/local-scan.test.ts',
+    'src/mcp/local-project-ports.test.ts',
+    'src/ingest/local-stage-ingest.test.ts',
+    'src/sl/pglite-sl-search-prototype.test.ts',
+    'src/core/git.service.test.ts',
+    'src/ingest/local-adapters.test.ts',
+    'src/ingest/local-bundle-ingest.test.ts',
+    'src/ingest/local-metabase-ingest.test.ts',
+    'src/sl/local-sl.test.ts',
+    'src/search/pglite-owner-process.test.ts',
+    'src/scan/local-enrichment-artifacts.test.ts',
+    'src/search/pglite-spike.test.ts',
+    'src/wiki/local-knowledge.test.ts',
+    'src/sl/local-query.test.ts',
+    'src/scan/relationship-review-decisions.test.ts',
+    'src/scan/relationship-profiling.test.ts',
+  ];
+
+  it('keeps slow package tests out of default local package test scripts', async () => {
+    const cliPackage = await readJson('../packages/cli/package.json');
+    const contextPackage = await readJson('../packages/context/package.json');
+
+    assertScriptContainsAll(cliPackage.scripts.test, cliSlowTests.map((file) => `--exclude ${file}`));
+    assertScriptContainsAll(contextPackage.scripts.test, contextSlowTests.map((file) => `--exclude ${file}`));
+    assert.match(contextPackage.scripts.test, /--exclude src\/scan\/relationship-benchmarks\.test\.ts/);
+  });
+
+  it('provides explicit slow package test scripts for CI', async () => {
+    const rootPackage = await readJson('../package.json');
+    const cliPackage = await readJson('../packages/cli/package.json');
+    const contextPackage = await readJson('../packages/context/package.json');
+
+    assert.equal(rootPackage.scripts['test:slow'], 'pnpm --filter @ktx/context run test:slow && pnpm --filter @ktx/cli run test:slow');
+    assertScriptContainsAll(cliPackage.scripts['test:slow'], cliSlowTests);
+    assertScriptContainsAll(contextPackage.scripts['test:slow'], contextSlowTests);
+    assert.doesNotMatch(contextPackage.scripts['test:slow'], /relationship-benchmarks\.test\.ts/);
+  });
+});