diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6732cf13..95fdf9d4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -24,6 +24,7 @@ on: permissions: contents: write + id-token: write concurrency: group: ktx-release-${{ github.ref }} @@ -68,19 +69,59 @@ jobs: - name: Install Python dependencies run: uv sync --all-packages + - name: Prepare next prerelease branch + if: ${{ inputs.release_kind == 'rc' }} + run: | + set -euo pipefail + + source_sha="$(git rev-parse HEAD)" + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + + if git ls-remote --exit-code --heads origin "${KTX_PRERELEASE_BRANCH}" >/dev/null 2>&1; then + git fetch origin "${KTX_PRERELEASE_BRANCH}" + git checkout -B "${KTX_PRERELEASE_BRANCH}" "origin/${KTX_PRERELEASE_BRANCH}" + git merge --no-edit "${source_sha}" + else + git checkout -B "${KTX_PRERELEASE_BRANCH}" "${source_sha}" + fi + + git push --set-upstream origin "HEAD:${KTX_PRERELEASE_BRANCH}" + env: + KTX_PRERELEASE_BRANCH: next + - name: Dry-run semantic release if: ${{ !inputs.publish_live }} - run: pnpm run semantic-release:dry-run + run: | + set -euo pipefail + + if [ "${KTX_RELEASE_KIND}" = "rc" ]; then + export GITHUB_REF="refs/heads/${KTX_PRERELEASE_BRANCH}" + export GITHUB_REF_NAME="${KTX_PRERELEASE_BRANCH}" + export GITHUB_SHA="$(git rev-parse HEAD)" + fi + + pnpm run semantic-release:dry-run env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} KTX_RELEASE_KIND: ${{ inputs.release_kind }} + KTX_PRERELEASE_BRANCH: next FORCE_RELEASE: ${{ inputs.force_release }} - name: Create semantic release if: ${{ inputs.publish_live }} - run: pnpm run semantic-release + run: | + set -euo pipefail + + if [ "${KTX_RELEASE_KIND}" = "rc" ]; then + export GITHUB_REF="refs/heads/${KTX_PRERELEASE_BRANCH}" + export GITHUB_REF_NAME="${KTX_PRERELEASE_BRANCH}" + export GITHUB_SHA="$(git rev-parse HEAD)" + fi + + pnpm run semantic-release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} KTX_RELEASE_KIND: ${{ inputs.release_kind }} + KTX_PRERELEASE_BRANCH: next FORCE_RELEASE: ${{ inputs.force_release }} - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/docs/release.md b/docs/release.md index 670a6a70..9131d57e 100644 --- a/docs/release.md +++ b/docs/release.md @@ -12,6 +12,11 @@ KTX has two npm release channels: - `rc` publishes prereleases such as `0.1.0-rc.2` to the npm `next` tag. - `stable` publishes normal releases such as `0.1.0` to the npm `latest` tag. +Run rc releases from the source branch you want to publish. The workflow +creates or updates the `next` prerelease branch from that source branch before +running semantic-release, because semantic-release requires a dedicated +prerelease branch in addition to the stable `main` branch. + Run stable releases only from `main`. The workflow rejects stable releases from other branches. @@ -19,10 +24,11 @@ other branches. Before you publish, confirm these requirements: -- The repository has an Actions secret named `NPM_TOKEN`. -- `NPM_TOKEN` is a granular npm token that can publish `@kaelio/ktx`. -- The token can publish non-interactively if the npm account or package uses - two-factor authentication for writes. +- npm Trusted Publishing is configured for `@kaelio/ktx`. +- The trusted publisher points at the `Kaelio/ktx` repository and the + `.github/workflows/release.yml` workflow. +- The workflow keeps `id-token: write` permission so npm can verify the + GitHub Actions run through OpenID Connect. - The repository has a baseline semantic-release tag for the latest published package version, such as `v0.1.0-rc.1`. @@ -43,8 +49,9 @@ publishing to npm. if semantic-release doesn't find a releasable commit. 7. Run the workflow. -The dry-run uses the same semantic-release configuration as a live release. It -doesn't publish to npm and doesn't commit release files. +The dry-run uses the same semantic-release configuration as a live release. For +rc releases, it can create or update the `next` branch. It doesn't publish to +npm and doesn't commit release files. ## Publish an rc release @@ -53,15 +60,16 @@ promoting to `latest`. 1. Open **Actions** in GitHub. 2. Select **KTX Release**. -3. Select the branch to release from. +3. Select the source branch to release from. 4. Set **release_kind** to `rc`. 5. Set **publish_live** to `true`. 6. Optional: Set **force_release** to `true`. 7. Run the workflow. -The workflow publishes `@kaelio/ktx` with `--access public --tag next`, runs the -published package smoke test, creates a GitHub release, and commits -`CHANGELOG.md`, `package.json`, and `release-policy.json`. +The workflow merges the selected source branch into `next`, publishes +`@kaelio/ktx` with `--access public --tag next`, runs the published package +smoke test, creates a GitHub release, and commits `CHANGELOG.md`, +`package.json`, and `release-policy.json` on `next`. ## Publish a stable release @@ -92,8 +100,11 @@ The artifact packaging and readiness scripts read `publicNpmPackageVersion` from `release-policy.json`, so manual version edits in build scripts aren't needed for rc releases. -## Trusted Publishing follow-up +## npm authentication -This workflow uses `NPM_TOKEN` today. Move to npm Trusted Publishing after the -final publish command path is verified for the package manager and workflow -filename configured in npm package settings. +The release workflow publishes through npm Trusted Publishing. It doesn't use +an `NPM_TOKEN` secret, and the publish step doesn't set `NODE_AUTH_TOKEN`. + +If npm returns an authentication error, check the Trusted Publishing settings +for the `@kaelio/ktx` package before adding token-based authentication back to +the workflow. diff --git a/packages/context/src/core/git-env.ts b/packages/context/src/core/git-env.ts index 7952d9c2..9ad3f121 100644 --- a/packages/context/src/core/git-env.ts +++ b/packages/context/src/core/git-env.ts @@ -25,5 +25,5 @@ function sanitizedGitEnv(env: NodeJS.ProcessEnv = process.env): NodeJS.ProcessEn } export function createSimpleGit(baseDir: string): SimpleGit { - return simpleGit({ baseDir }).env(sanitizedGitEnv()); + return simpleGit({ baseDir, unsafe: { allowUnsafeAskPass: true } }).env(sanitizedGitEnv()); } diff --git a/packages/context/src/core/git.service.test.ts b/packages/context/src/core/git.service.test.ts index 8ad74b22..e8a5aa73 100644 --- a/packages/context/src/core/git.service.test.ts +++ b/packages/context/src/core/git.service.test.ts @@ -59,6 +59,34 @@ describe('GitService', () => { expect(config).toMatch(/\[gc]\n\s+autoDetach = false/); expect(config).toMatch(/\[maintenance]\n\s+autoDetach = false/); }); + + it('initializes when release automation sets GIT_ASKPASS', async () => { + const releaseEnvDir = await mkdtemp(join(tmpdir(), 'git-service-release-env-')); + const previousAskPass = process.env.GIT_ASKPASS; + process.env.GIT_ASKPASS = 'echo'; + + try { + const releaseEnvService = new GitService({ + storage: { configDir: releaseEnvDir, homeDir: releaseEnvDir }, + git: { + userName: 'Test User', + userEmail: 'test@example.com', + bootstrapMessage: 'Initialize test config repo', + bootstrapAuthor: 'test-system', + bootstrapAuthorEmail: 'system@example.com', + }, + }); + + await expect(releaseEnvService.onModuleInit()).resolves.toBeUndefined(); + } finally { + if (previousAskPass === undefined) { + delete process.env.GIT_ASKPASS; + } else { + process.env.GIT_ASKPASS = previousAskPass; + } + await rm(releaseEnvDir, { recursive: true, force: true }); + } + }); }); describe('commitFile `created` flag', () => { diff --git a/packages/context/src/ingest/git-env.ts b/packages/context/src/ingest/git-env.ts index 25a5f021..98590536 100644 --- a/packages/context/src/ingest/git-env.ts +++ b/packages/context/src/ingest/git-env.ts @@ -3,6 +3,7 @@ import { type SimpleGit, simpleGit } from 'simple-git'; const SANITIZED_GIT_ENV_KEYS = [ 'EDITOR', 'GIT_ALTERNATE_OBJECT_DIRECTORIES', + 'GIT_ASKPASS', 'GIT_CONFIG', 'GIT_CONFIG_COUNT', 'GIT_CONFIG_GLOBAL', @@ -31,5 +32,5 @@ export function createSimpleGit(baseDir?: string): SimpleGit { for (const key of SANITIZED_GIT_ENV_KEYS) { delete env[key]; } - return simpleGit(baseDir).env(env); + return simpleGit({ baseDir, unsafe: { allowUnsafeAskPass: true } }).env(env); } diff --git a/scripts/publish-public-npm-package.mjs b/scripts/publish-public-npm-package.mjs index d82a157d..c8c5b6e3 100644 --- a/scripts/publish-public-npm-package.mjs +++ b/scripts/publish-public-npm-package.mjs @@ -1,14 +1,13 @@ #!/usr/bin/env node -import { execFile } from 'node:child_process'; +import { spawn } from 'node:child_process'; import { access } from 'node:fs/promises'; import { pathToFileURL } from 'node:url'; -import { promisify } from 'node:util'; import { packageArtifactLayout } from './package-artifacts.mjs'; import { releaseReadinessReport } from './release-readiness.mjs'; -const execFileAsync = promisify(execFile); +export const NPM_PUBLISH_TIMEOUT_MS = 180_000; export function resolvePublishMode(args = process.argv.slice(2)) { return { live: args.includes('--publish') }; @@ -23,7 +22,7 @@ export function requireNpmPublicReleaseReady(report) { export function buildNpmPublishCommand(tarballPath, publish, mode) { return { - command: 'pnpm', + command: 'npm', args: [ 'publish', tarballPath, @@ -31,7 +30,7 @@ export function buildNpmPublishCommand(tarballPath, publish, mode) { publish.access, '--tag', publish.tag, - ...(mode.live ? [] : ['--dry-run', '--no-git-checks']), + ...(mode.live ? [] : ['--dry-run']), ], env: publish.registry ? { npm_config_registry: publish.registry } : {}, }; @@ -47,10 +46,42 @@ async function assertFileExists(path) { async function runPublishCommand(command) { process.stdout.write(`$ ${command.command} ${command.args.join(' ')}\n`); - await execFileAsync(command.command, command.args, { - env: { ...process.env, ...command.env }, - encoding: 'utf8', - maxBuffer: 1024 * 1024 * 20, + + await new Promise((resolvePromise, reject) => { + let settled = false; + const child = spawn(command.command, command.args, { + env: { ...process.env, ...command.env }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + const settle = (callback, value) => { + if (settled) { + return; + } + settled = true; + clearTimeout(timeout); + callback(value); + }; + const timeout = setTimeout(() => { + child.kill('SIGTERM'); + settle(reject, new Error(`Timed out after ${NPM_PUBLISH_TIMEOUT_MS}ms while publishing npm package`)); + }, NPM_PUBLISH_TIMEOUT_MS); + + child.stdout.on('data', (chunk) => { + process.stdout.write(chunk); + }); + child.stderr.on('data', (chunk) => { + process.stderr.write(chunk); + }); + child.on('error', (error) => { + settle(reject, error); + }); + child.on('close', (code, signal) => { + if (code === 0) { + settle(resolvePromise); + return; + } + settle(reject, new Error(`npm publish failed with ${signal ? `signal ${signal}` : `exit code ${code}`}`)); + }); }); } diff --git a/scripts/publish-public-npm-package.test.mjs b/scripts/publish-public-npm-package.test.mjs index d622ffe2..d3b5b016 100644 --- a/scripts/publish-public-npm-package.test.mjs +++ b/scripts/publish-public-npm-package.test.mjs @@ -49,13 +49,13 @@ describe('requireNpmPublicReleaseReady', () => { }); describe('buildNpmPublishCommand', () => { - it('builds a dry-run pnpm publish command by default', () => { + it('builds a dry-run npm publish command by default', () => { assert.deepEqual( buildNpmPublishCommand('/repo/ktx/dist/artifacts/npm/kaelio-ktx-0.1.0-rc.1.tgz', readyReport.npmPublish, { live: false, }), { - command: 'pnpm', + command: 'npm', args: [ 'publish', '/repo/ktx/dist/artifacts/npm/kaelio-ktx-0.1.0-rc.1.tgz', @@ -64,7 +64,6 @@ describe('buildNpmPublishCommand', () => { '--tag', 'next', '--dry-run', - '--no-git-checks', ], env: {}, }, diff --git a/scripts/published-package-smoke.mjs b/scripts/published-package-smoke.mjs index e304a7af..88902c42 100644 --- a/scripts/published-package-smoke.mjs +++ b/scripts/published-package-smoke.mjs @@ -2,7 +2,7 @@ import assert from 'node:assert/strict'; import { execFile } from 'node:child_process'; -import { mkdtemp, rm } from 'node:fs/promises'; +import { mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { dirname, join, resolve } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; @@ -22,6 +22,8 @@ export { const execFileAsync = promisify(execFile); const SMOKE_TIMEOUT_MS = 180_000; +const TRANSIENT_LOOKUP_RETRY_ATTEMPTS = 6; +const TRANSIENT_LOOKUP_RETRY_DELAY_MS = 10_000; const VERSION_LABELS = new Set([ 'published package npx version', @@ -33,6 +35,18 @@ export function isPublishedPackageVersionLabel(label) { return VERSION_LABELS.has(label); } +export function publishedPackageSmokePnpmWorkspaceYaml() { + return ['packages:', ' - "."', 'allowBuilds:', ' better-sqlite3: true', ''].join('\n'); +} + +export function isTransientPublishedPackageLookupFailure(result) { + return ( + result.code !== 0 && + (result.stderr.includes('npm error code ETARGET') || + result.stderr.includes('No matching version found for @kaelio/ktx@')) + ); +} + function scriptRootDir() { return resolve(dirname(fileURLToPath(import.meta.url)), '..'); } @@ -61,6 +75,24 @@ async function runCommand(command, args, options = {}) { } } +function delay(ms) { + return new Promise((resolvePromise) => setTimeout(resolvePromise, ms)); +} + +async function runCommandWithRegistryRetry(command, args, options = {}) { + const attempts = options.retryAttempts ?? TRANSIENT_LOOKUP_RETRY_ATTEMPTS; + const retryDelayMs = options.retryDelayMs ?? TRANSIENT_LOOKUP_RETRY_DELAY_MS; + let result = await runCommand(command, args, options); + + for (let attempt = 2; attempt <= attempts && isTransientPublishedPackageLookupFailure(result); attempt += 1) { + process.stdout.write(`npm registry has not exposed the package yet; retrying smoke command (${attempt}/${attempts})\n`); + await delay(retryDelayMs); + result = await runCommand(command, args, options); + } + + return result; +} + function requireSuccess(label, result) { assert.equal( result.code, @@ -74,15 +106,17 @@ export async function runPublishedPackageSmoke(config) { try { const projectDir = join(root, 'demo-project'); + await writeFile(join(root, 'pnpm-workspace.yaml'), publishedPackageSmokePnpmWorkspaceYaml()); + const commands = buildPublishedPackageSmokeCommands(config, projectDir); const pnpmHome = join(root, 'pnpm-home'); const globalEnv = { PNPM_HOME: pnpmHome, - PATH: `${pnpmHome}${process.platform === 'win32' ? ';' : ':'}${process.env.PATH ?? ''}`, + PATH: [join(pnpmHome, 'bin'), pnpmHome, process.env.PATH ?? ''].join(process.platform === 'win32' ? ';' : ':'), }; for (const command of commands) { const isGlobalCommand = command.label.includes('global'); - const result = await runCommand(command.command, command.args, { + const result = await runCommandWithRegistryRetry(command.command, command.args, { cwd: command.label.includes('local') || isGlobalCommand ? root : undefined, env: isGlobalCommand ? { ...globalEnv, ...command.env } : command.env, }); diff --git a/scripts/published-package-smoke.test.mjs b/scripts/published-package-smoke.test.mjs index 56bbe424..719eed2a 100644 --- a/scripts/published-package-smoke.test.mjs +++ b/scripts/published-package-smoke.test.mjs @@ -6,6 +6,8 @@ import { buildPublishedPackageNpxCommand, buildPublishedPackageSmokeCommands, isPublishedPackageVersionLabel, + isTransientPublishedPackageLookupFailure, + publishedPackageSmokePnpmWorkspaceYaml, publishedPackageSpec, readPublishedPackageSmokeConfig, } from './published-package-smoke.mjs'; @@ -156,6 +158,33 @@ describe('published package smoke output validation labels', () => { }); }); +describe('published package smoke registry retry classification', () => { + it('recognizes npm propagation misses as transient lookup failures', () => { + assert.equal( + isTransientPublishedPackageLookupFailure({ + code: 1, + stdout: '', + stderr: [ + 'npm error code ETARGET', + 'npm error notarget No matching version found for @kaelio/ktx@0.1.0-rc.4.', + ].join('\n'), + }), + true, + ); + }); + + it('does not retry unrelated command failures', () => { + assert.equal( + isTransientPublishedPackageLookupFailure({ + code: 1, + stdout: '', + stderr: 'npm error code EOTP', + }), + false, + ); + }); +}); + describe('published package smoke command construction', () => { const config = { enabled: true, @@ -244,6 +273,13 @@ describe('published package smoke command construction', () => { ); }); + it('allows native dependency build scripts in clean pnpm smoke installs', () => { + assert.equal( + publishedPackageSmokePnpmWorkspaceYaml(), + ['packages:', ' - "."', 'allowBuilds:', ' better-sqlite3: true', ''].join('\n'), + ); + }); + it('exposes the smoke through the package release script', async () => { const packageJson = JSON.parse(await readFile(new URL('../package.json', import.meta.url), 'utf8')); diff --git a/scripts/release-workflow.test.mjs b/scripts/release-workflow.test.mjs index 8e8d6df0..60a82011 100644 --- a/scripts/release-workflow.test.mjs +++ b/scripts/release-workflow.test.mjs @@ -14,13 +14,18 @@ describe('release workflow', () => { assert.match(workflow, /publish_live:/); assert.match(workflow, /default: false/); assert.match(workflow, /^ contents: write$/m); + assert.match(workflow, /^ id-token: write$/m); assert.match(workflow, /fetch-depth: 0/); assert.match(workflow, /registry-url: "https:\/\/registry\.npmjs\.org"/); + assert.match(workflow, /Prepare next prerelease branch/); + assert.match(workflow, /git checkout -B "\$\{KTX_PRERELEASE_BRANCH\}"/); + assert.match(workflow, /GITHUB_REF="refs\/heads\/\$\{KTX_PRERELEASE_BRANCH\}"/); assert.match(workflow, /pnpm run semantic-release:dry-run/); assert.match(workflow, /pnpm run semantic-release$/m); assert.match(workflow, /KTX_RELEASE_KIND: \$\{\{ inputs.release_kind \}\}/); + assert.match(workflow, /KTX_PRERELEASE_BRANCH: next/); assert.match(workflow, /FORCE_RELEASE: \$\{\{ inputs.force_release \}\}/); - assert.match(workflow, /NODE_AUTH_TOKEN: \$\{\{ secrets.NPM_TOKEN \}\}/); + assert.doesNotMatch(workflow, /NODE_AUTH_TOKEN/); assert.doesNotMatch(workflow, /^ push:/m); assert.doesNotMatch(workflow, /^ pull_request:/m); }); diff --git a/scripts/semantic-release-config.cjs b/scripts/semantic-release-config.cjs index 85df2620..66b9aad7 100644 --- a/scripts/semantic-release-config.cjs +++ b/scripts/semantic-release-config.cjs @@ -82,6 +82,10 @@ function releaseKind(env) { return env.KTX_RELEASE_KIND || env.INPUT_RELEASE_KIND || 'rc'; } +function prereleaseBranch(env) { + return env.KTX_PRERELEASE_BRANCH || env.INPUT_PRERELEASE_BRANCH || 'next'; +} + function releaseTag(kind) { return kind === 'rc' ? 'next' : 'latest'; } @@ -91,7 +95,7 @@ function releaseBranches(env = process.env) { const kind = releaseKind(env); if (kind === 'rc') { - return [{ name: branch, prerelease: 'rc', channel: 'next' }]; + return ['main', { name: prereleaseBranch(env), prerelease: 'rc', channel: 'next' }]; } if (kind === 'stable') { @@ -170,6 +174,7 @@ function createReleaseConfig(env = process.env) { module.exports = { createReleaseConfig, + prereleaseBranch, releaseBranches, releaseKind, releaseTag, diff --git a/scripts/semantic-release-config.test.mjs b/scripts/semantic-release-config.test.mjs index f61d3498..0a267d77 100644 --- a/scripts/semantic-release-config.test.mjs +++ b/scripts/semantic-release-config.test.mjs @@ -10,14 +10,15 @@ function releaseExecOptions(config) { } describe('semantic-release config', () => { - it('configures manual rc releases on the selected branch with next channel', () => { + it('configures rc releases on a dedicated next prerelease branch', () => { assert.equal(releaseKind({ KTX_RELEASE_KIND: 'rc' }), 'rc'); assert.equal(releaseTag('rc'), 'next'); - assert.deepEqual(releaseBranches({ KTX_RELEASE_KIND: 'rc', GITHUB_REF_NAME: 'release-candidate' }), [ - { name: 'release-candidate', prerelease: 'rc', channel: 'next' }, + assert.deepEqual(releaseBranches({ KTX_RELEASE_KIND: 'rc', GITHUB_REF_NAME: 'main' }), [ + 'main', + { name: 'next', prerelease: 'rc', channel: 'next' }, ]); - const config = createReleaseConfig({ KTX_RELEASE_KIND: 'rc', GITHUB_REF_NAME: 'release-candidate' }); + const config = createReleaseConfig({ KTX_RELEASE_KIND: 'rc', GITHUB_REF_NAME: 'main' }); assert.match( releaseExecOptions(config).prepareCmd, /update-public-release-version\.mjs "\$\{nextRelease\.version\}" "next"/,