diff --git a/README.md b/README.md index 19c82853..eb78b1a3 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,10 @@ Use KTX when you want agents to: Supports PostgreSQL, Snowflake, BigQuery, ClickHouse, MySQL, SQL Server, and SQLite. +

+ KTX ingestion flow from source systems through validation to wiki and semantic-layer outputs +

+ ## Agent Setup Ask an agent such as Claude Code, Codex, Cursor, or OpenCode to install and diff --git a/docs-site/public/images/ingestion-flow-transparent.svg b/docs-site/public/images/ingestion-flow-transparent.svg new file mode 100644 index 00000000..6fa1a40a --- /dev/null +++ b/docs-site/public/images/ingestion-flow-transparent.svg @@ -0,0 +1,211 @@ + + KTX ingestion flow + Source systems flow through source adapters, context builder, reconciliation, and validation to create wiki Markdown and semantic-layer YAML outputs. + + + + + + + + + + + + + + + + + + + + + + + + + Databases + Schemas, columns, keys, + row counts, and query + history. + + + PostgreSQL + + Snowflake + + BigQuery + + SQLite + + + + + + + BI tools + Dashboards, questions, + explores, usage, and trusted + examples. + + + Metabase + + Looker + + + + + + + Modeling code + Existing metrics, dimensions, + models, joins, and entities. + + + dbt + + LookML + + MetricFlow + + + + + + + Docs and notes + Policies, caveats, team + definitions, and analyst + context. + + + Notion + + Any text + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1 + Source adapters + Read each configured system in + its native shape. + + + + + + 2 + Context builder + Turn source evidence into + proposed context updates. + + + + + + 3 + Reconciliation + Merge new evidence with the + context that already exists. + + + + + + 4 + Validation + Check references and + semantics before agents rely on + them. + + + + + + + + wiki/*.md + Wiki + + + free-form + + auto-maintained + + Definitions, caveats, policies, analyst notes, and + business language that agents can search. + + + + + + semantic-layer/*.yaml + Semantic layer + + + structured + + executable + + auto-maintained + + Metrics, joins, tables, dimensions, filters, and + segments that KTX can validate and compile into + SQL. + + + + + references + + + diff --git a/docs-site/public/images/ingestion-flow.png b/docs-site/public/images/ingestion-flow.png new file mode 100644 index 00000000..49bc544f Binary files /dev/null and b/docs-site/public/images/ingestion-flow.png differ diff --git a/docs/release.md b/docs/release.md index 833063d8..4042863c 100644 --- a/docs/release.md +++ b/docs/release.md @@ -3,7 +3,7 @@ This runbook covers the maintainer workflow for publishing `@kaelio/ktx` to npm through GitHub Actions. The workflow uses semantic-release to choose the next version, update release metadata, publish the package, create the GitHub -release, and commit the release files back to the repository. +release, and commit prerelease files back to the `next` branch. ## Release channels @@ -101,8 +101,9 @@ Publish a stable release from `main` after you have validated an rc package. 7. Run the workflow. The workflow publishes `@kaelio/ktx` with `--access public --tag latest`, runs -the published package smoke test, creates a GitHub release, and commits the -release metadata. +the published package smoke test, and creates a GitHub release. Stable releases +don't commit release metadata back to `main`, because `main` is protected and +requires changes through pull requests. ## Release metadata @@ -118,7 +119,8 @@ The artifact packaging and readiness scripts read `publicNpmPackageVersion` from `release-policy.json`, so manual version edits in build scripts aren't needed for rc releases. The semantic-release npm plugin publishes the generated `dist/public-npm-package` tree and writes the release tarball under -`dist/artifacts/npm`. +`dist/artifacts/npm`. Stable releases use the updated metadata during the +workflow run, but that generated metadata isn't committed back to `main`. The bundled Python runtime wheel also derives its version from `publicNpmPackageVersion`. Stable npm versions are reused as-is, and rc diff --git a/packages/connector-clickhouse/src/package-exports.test.ts b/packages/connector-clickhouse/src/package-exports.test.ts index 96ba11d4..644e6075 100644 --- a/packages/connector-clickhouse/src/package-exports.test.ts +++ b/packages/connector-clickhouse/src/package-exports.test.ts @@ -1,12 +1,16 @@ import { describe, expect, it } from 'vitest'; describe('@ktx/connector-clickhouse package exports', () => { - it('exports public connector APIs during package bootstrap', async () => { - const connector = await import('./index.js'); + it( + 'exports public connector APIs during package bootstrap', + async () => { + const connector = await import('./index.js'); - expect(connector.KtxClickHouseDialect).toBeTypeOf('function'); - expect(connector.KtxClickHouseScanConnector).toBeTypeOf('function'); - expect(connector.clickHouseClientConfigFromConfig).toBeTypeOf('function'); - expect(connector.createClickHouseLiveDatabaseIntrospection).toBeTypeOf('function'); - }); + expect(connector.KtxClickHouseDialect).toBeTypeOf('function'); + expect(connector.KtxClickHouseScanConnector).toBeTypeOf('function'); + expect(connector.clickHouseClientConfigFromConfig).toBeTypeOf('function'); + expect(connector.createClickHouseLiveDatabaseIntrospection).toBeTypeOf('function'); + }, + 20_000, + ); }); diff --git a/scripts/build-public-npm-package.mjs b/scripts/build-public-npm-package.mjs index 0e34ae6d..63551d38 100644 --- a/scripts/build-public-npm-package.mjs +++ b/scripts/build-public-npm-package.mjs @@ -146,12 +146,12 @@ export function publicNpmPackageJson(cliPackageJson, dependencies, version = PUB license: cliPackageJson.license ?? 'Apache-2.0', repository: { type: 'git', - url: 'git+https://github.com/kaelio/ktx.git', + url: 'https://github.com/Kaelio/ktx', }, bugs: { - url: 'https://github.com/kaelio/ktx/issues', + url: 'https://github.com/Kaelio/ktx/issues', }, - homepage: 'https://github.com/kaelio/ktx#readme', + homepage: 'https://github.com/Kaelio/ktx#readme', }; } diff --git a/scripts/build-public-npm-package.test.mjs b/scripts/build-public-npm-package.test.mjs index 4afe1de5..c78ae164 100644 --- a/scripts/build-public-npm-package.test.mjs +++ b/scripts/build-public-npm-package.test.mjs @@ -217,6 +217,14 @@ describe('publicNpmPackageJson', () => { assert.deepEqual(packageJson.dependencies, { commander: '14.0.3' }); assert.deepEqual(packageJson.bundledDependencies, PUBLIC_BUNDLED_WORKSPACE_PACKAGES); assert.deepEqual(packageJson.files, ['dist', 'assets']); + assert.deepEqual(packageJson.repository, { + type: 'git', + url: 'https://github.com/Kaelio/ktx', + }); + assert.deepEqual(packageJson.bugs, { + url: 'https://github.com/Kaelio/ktx/issues', + }); + assert.equal(packageJson.homepage, 'https://github.com/Kaelio/ktx#readme'); }); }); diff --git a/scripts/semantic-release-config.cjs b/scripts/semantic-release-config.cjs index dfc837cc..e9ec9cc5 100644 --- a/scripts/semantic-release-config.cjs +++ b/scripts/semantic-release-config.cjs @@ -90,6 +90,26 @@ function releaseTag(kind) { return kind === 'rc' ? 'next' : 'latest'; } +function releaseChangelogPlugins(kind) { + return kind === 'rc' ? ['@semantic-release/changelog'] : []; +} + +function releaseGitPlugins(kind) { + if (kind !== 'rc') { + return []; + } + + return [ + [ + '@semantic-release/git', + { + assets: ['CHANGELOG.md', 'package.json', 'release-policy.json'], + message: 'chore(release): ${nextRelease.version} [skip ci]\n\n${nextRelease.notes}', + }, + ], + ]; +} + function releaseBranches(env = process.env) { const branch = currentBranch(env); const kind = releaseKind(env); @@ -137,7 +157,7 @@ function createReleaseConfig(env = process.env) { }, }, ], - '@semantic-release/changelog', + ...releaseChangelogPlugins(kind), [ '@semantic-release/exec', { @@ -161,13 +181,7 @@ function createReleaseConfig(env = process.env) { publishCmd: 'pnpm run release:published-smoke', }, ], - [ - '@semantic-release/git', - { - assets: ['CHANGELOG.md', 'package.json', 'release-policy.json'], - message: 'chore(release): ${nextRelease.version} [skip ci]\n\n${nextRelease.notes}', - }, - ], + ...releaseGitPlugins(kind), [ '@semantic-release/github', { diff --git a/scripts/semantic-release-config.test.mjs b/scripts/semantic-release-config.test.mjs index 109f114a..99cd53c4 100644 --- a/scripts/semantic-release-config.test.mjs +++ b/scripts/semantic-release-config.test.mjs @@ -9,6 +9,14 @@ function releaseExecOptions(config) { return config.plugins.find((plugin) => Array.isArray(plugin) && plugin[0] === '@semantic-release/exec' && plugin[1].prepareCmd)[1]; } +function releaseExecIndex(config) { + return config.plugins.findIndex((plugin) => Array.isArray(plugin) && plugin[0] === '@semantic-release/exec' && plugin[1].prepareCmd); +} + +function pluginNames(config) { + return config.plugins.map((plugin) => (Array.isArray(plugin) ? plugin[0] : plugin)); +} + describe('semantic-release config', () => { it('configures rc releases on a dedicated next prerelease branch', () => { assert.equal(releaseKind({ KTX_RELEASE_KIND: 'rc' }), 'rc'); @@ -33,7 +41,15 @@ describe('semantic-release config', () => { releaseExecOptions(config).prepareCmd, /update-public-release-version\.mjs "\$\{nextRelease\.version\}" "next"/, ); - assert.doesNotMatch(releaseExecOptions(config).publishCmd ?? '', /release:npm-publish/); + assert.doesNotMatch(JSON.stringify(config.plugins), /release:npm-publish/); + const releaseFilePluginNames = pluginNames(config).filter( + (plugin) => plugin === '@semantic-release/changelog' || plugin === '@semantic-release/git', + ); + assert.deepEqual(releaseFilePluginNames, ['@semantic-release/changelog', '@semantic-release/git']); + + const names = pluginNames(config); + assert.ok(names.indexOf('@semantic-release/changelog') < releaseExecIndex(config)); + assert.ok(names.indexOf('@semantic-release/git') > releaseExecIndex(config)); }); it('configures stable releases only from main with latest tag', () => { @@ -49,6 +65,13 @@ describe('semantic-release config', () => { assert.equal(config.plugins.includes('./scripts/semantic-release-version-policy.cjs'), false); }); + it('does not commit release files back to protected main during stable releases', () => { + const config = createReleaseConfig({ KTX_RELEASE_KIND: 'stable', GITHUB_REF_NAME: 'main' }); + + assert.equal(pluginNames(config).includes('@semantic-release/git'), false); + assert.equal(pluginNames(config).includes('@semantic-release/changelog'), false); + }); + it('rejects stable releases from non-main branches', () => { assert.throws( () => releaseBranches({ KTX_RELEASE_KIND: 'stable', GITHUB_REF_NAME: 'feature/release-test' }),