mirror of
https://github.com/Kaelio/ktx.git
synced 2026-07-16 11:31:02 +02:00
The 34 `index.ts` re-export barrels inside `packages/cli/src/` were holdovers from the pre-fold multi-workspace structure. Post-fold-in they served no production purpose: external consumers go through the single package main entry, and in-repo callers mostly imported through them only because the path was short. Internally, knip flagged most barrel re-exports as production-dead (only reached via tests). This change: - Deletes every internal barrel except `packages/cli/src/index.ts` (the published package entry). - Rewrites ~270 source/test files to import each name directly from the file that defines it. - Moves `tools/warehouse-verification/index.ts` to `create-warehouse-verification-tools.ts` (the function it defined locally) and updates its single consumer. - Renames `search/backend-conformance.ts` → `.test-utils.ts` to match the existing test-helper file convention. - Deletes 13 dead test-only chains (dbt-descriptions/*, live-database/extracted-schema, live-database/structural-sync, relationship-* feedback/review chain) plus their tests and a cascading orphan integration test. - Updates test mocks that pointed at deleted barrel paths (notion-client, connector barrels in scan/local-scan-connectors tests) to mock the source files instead. - Points the maintainer benchmark script (`scripts/relationship-benchmark-report.mjs`) at source files instead of `dist/context/scan/index.js`. - Drops the barrel `!` entries from `knip.json`; adds explicit production entries only for the benchmark code reached via dist by the maintainer script. Net: 413 files changed, ~1.2k insertions, ~9.4k deletions. `pnpm run dead-code` (Biome + knip default + knip production) and `pnpm run type-check` are clean; 2277 tests pass.
224 lines
7.7 KiB
TypeScript
224 lines
7.7 KiB
TypeScript
import { readFile } from 'node:fs/promises';
|
|
import { homedir } from 'node:os';
|
|
import { resolve } from 'node:path';
|
|
import {
|
|
NOTION_DEFAULT_MAX_KNOWLEDGE_CREATES_PER_RUN,
|
|
type NotionPullConfig,
|
|
notionPullConfigSchema,
|
|
} from '../ingest/adapters/notion/types.js';
|
|
import type { KtxProjectConnectionConfig } from '../project/config.js';
|
|
|
|
const KTX_NOTION_ORG_KNOWLEDGE_WARNING =
|
|
'Anything accessible to this Notion integration can become organization knowledge.';
|
|
|
|
type KtxNotionCrawlMode = 'all_accessible' | 'selected_roots';
|
|
|
|
type RawKtxNotionConnectionConfig = Extract<KtxProjectConnectionConfig, { driver: 'notion' }>;
|
|
|
|
export type KtxNotionConnectionConfig = Omit<
|
|
RawKtxNotionConnectionConfig,
|
|
| 'auth_token'
|
|
| 'auth_token_ref'
|
|
| 'crawl_mode'
|
|
| 'root_page_ids'
|
|
| 'root_database_ids'
|
|
| 'root_data_source_ids'
|
|
| 'max_pages_per_run'
|
|
| 'max_knowledge_creates_per_run'
|
|
| 'max_knowledge_updates_per_run'
|
|
> & {
|
|
driver: 'notion';
|
|
auth_token: string | null;
|
|
auth_token_ref: string | null;
|
|
crawl_mode: KtxNotionCrawlMode;
|
|
root_page_ids: string[];
|
|
root_database_ids: string[];
|
|
root_data_source_ids: string[];
|
|
max_pages_per_run: number;
|
|
max_knowledge_creates_per_run: number;
|
|
max_knowledge_updates_per_run: number;
|
|
};
|
|
|
|
/** @internal */
|
|
export interface RedactedKtxNotionConnectionConfig {
|
|
driver: 'notion';
|
|
hasAuthToken: boolean;
|
|
crawlMode: KtxNotionCrawlMode;
|
|
rootPageIds: string[];
|
|
rootDatabaseIds: string[];
|
|
rootDataSourceIds: string[];
|
|
maxPagesPerRun: number;
|
|
maxKnowledgeCreatesPerRun: number;
|
|
maxKnowledgeUpdatesPerRun: number;
|
|
warning: typeof KTX_NOTION_ORG_KNOWLEDGE_WARNING;
|
|
}
|
|
|
|
interface ResolveNotionTokenOptions {
|
|
env?: Record<string, string | undefined>;
|
|
readTextFile?: (path: string) => Promise<string>;
|
|
}
|
|
|
|
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
}
|
|
|
|
function record(value: unknown): Record<string, unknown> {
|
|
if (!isRecord(value)) {
|
|
throw new Error('Notion connection config must be an object');
|
|
}
|
|
return value;
|
|
}
|
|
|
|
function stringValue(value: unknown, fallback: string): string {
|
|
return typeof value === 'string' && value.trim().length > 0 ? value.trim() : fallback;
|
|
}
|
|
|
|
function optionalString(value: unknown): string | null {
|
|
return typeof value === 'string' && value.trim().length > 0 ? value.trim() : null;
|
|
}
|
|
|
|
function stringArray(value: unknown): string[] {
|
|
if (!Array.isArray(value)) {
|
|
return [];
|
|
}
|
|
return value.filter((item): item is string => typeof item === 'string' && item.trim().length > 0);
|
|
}
|
|
|
|
function integerWithFallback(value: unknown, fallback: number, name: string): number {
|
|
if (value === undefined || value === null) {
|
|
return fallback;
|
|
}
|
|
if (typeof value !== 'number' || !Number.isInteger(value)) {
|
|
throw new Error(`${name} must be an integer`);
|
|
}
|
|
return value;
|
|
}
|
|
|
|
function boundedInteger(value: unknown, fallback: number, name: string, min: number, max: number): number {
|
|
const parsed = integerWithFallback(value, fallback, name);
|
|
if (parsed < min || parsed > max) {
|
|
throw new Error(`${name} must be between ${min} and ${max}`);
|
|
}
|
|
return parsed;
|
|
}
|
|
|
|
export function parseNotionConnectionConfig(raw: unknown): KtxNotionConnectionConfig {
|
|
const input = record(raw);
|
|
if (input.driver !== 'notion') {
|
|
throw new Error('Notion connection config requires driver: notion');
|
|
}
|
|
const authToken = optionalString(input.auth_token);
|
|
const authTokenRef = optionalString(input.auth_token_ref);
|
|
if (!authToken && !authTokenRef) {
|
|
throw new Error('Notion connection config requires auth_token or auth_token_ref');
|
|
}
|
|
if (authTokenRef && !authTokenRef.startsWith('env:') && !authTokenRef.startsWith('file:')) {
|
|
throw new Error('Notion auth_token_ref must use env:NAME or file:/path');
|
|
}
|
|
|
|
const crawlMode = stringValue(input.crawl_mode, 'selected_roots');
|
|
if (crawlMode !== 'selected_roots' && crawlMode !== 'all_accessible') {
|
|
throw new Error(`Unsupported Notion crawl_mode: ${crawlMode}`);
|
|
}
|
|
const rootPageIds = stringArray(input.root_page_ids);
|
|
const rootDatabaseIds = stringArray(input.root_database_ids);
|
|
const rootDataSourceIds = stringArray(input.root_data_source_ids);
|
|
if (crawlMode === 'selected_roots' && rootPageIds.length + rootDatabaseIds.length + rootDataSourceIds.length === 0) {
|
|
throw new Error('selected_roots requires at least one root page, database, or data source id');
|
|
}
|
|
|
|
return {
|
|
driver: 'notion',
|
|
auth_token: authToken,
|
|
auth_token_ref: authTokenRef,
|
|
crawl_mode: crawlMode,
|
|
root_page_ids: rootPageIds,
|
|
root_database_ids: rootDatabaseIds,
|
|
root_data_source_ids: rootDataSourceIds,
|
|
max_pages_per_run: boundedInteger(input.max_pages_per_run, 1000, 'max_pages_per_run', 1, 10_000),
|
|
max_knowledge_creates_per_run: boundedInteger(
|
|
input.max_knowledge_creates_per_run,
|
|
NOTION_DEFAULT_MAX_KNOWLEDGE_CREATES_PER_RUN,
|
|
'max_knowledge_creates_per_run',
|
|
0,
|
|
25,
|
|
),
|
|
max_knowledge_updates_per_run: boundedInteger(
|
|
input.max_knowledge_updates_per_run,
|
|
20,
|
|
'max_knowledge_updates_per_run',
|
|
0,
|
|
100,
|
|
),
|
|
};
|
|
}
|
|
|
|
/** @internal */
|
|
export function redactNotionConnectionConfig(config: KtxNotionConnectionConfig): RedactedKtxNotionConnectionConfig {
|
|
return {
|
|
driver: 'notion',
|
|
hasAuthToken: Boolean(config.auth_token ?? config.auth_token_ref),
|
|
crawlMode: config.crawl_mode,
|
|
rootPageIds: config.root_page_ids,
|
|
rootDatabaseIds: config.root_database_ids,
|
|
rootDataSourceIds: config.root_data_source_ids,
|
|
maxPagesPerRun: config.max_pages_per_run,
|
|
maxKnowledgeCreatesPerRun: config.max_knowledge_creates_per_run,
|
|
maxKnowledgeUpdatesPerRun: config.max_knowledge_updates_per_run,
|
|
warning: KTX_NOTION_ORG_KNOWLEDGE_WARNING,
|
|
};
|
|
}
|
|
|
|
function expandHome(path: string): string {
|
|
return path === '~' || path.startsWith('~/') ? resolve(homedir(), path.slice(2)) : path;
|
|
}
|
|
|
|
/** @internal */
|
|
export async function resolveNotionAuthToken(
|
|
authTokenRef: string,
|
|
options: ResolveNotionTokenOptions = {},
|
|
): Promise<string> {
|
|
if (authTokenRef.startsWith('env:')) {
|
|
const envName = authTokenRef.slice('env:'.length);
|
|
const value = (options.env ?? process.env)[envName];
|
|
if (!value) {
|
|
throw new Error(`Notion token environment variable ${envName} is not set`);
|
|
}
|
|
return value.trim();
|
|
}
|
|
if (authTokenRef.startsWith('file:')) {
|
|
const path = expandHome(authTokenRef.slice('file:'.length));
|
|
const readTextFile = options.readTextFile ?? ((filePath: string) => readFile(filePath, 'utf-8'));
|
|
const value = (await readTextFile(path)).trim();
|
|
if (!value) {
|
|
throw new Error(`Notion token file is empty: ${path}`);
|
|
}
|
|
return value;
|
|
}
|
|
throw new Error('Notion auth_token_ref must use env:NAME or file:/path');
|
|
}
|
|
|
|
export async function resolveNotionConnectionAuthToken(
|
|
config: Pick<KtxNotionConnectionConfig, 'auth_token' | 'auth_token_ref'>,
|
|
options: ResolveNotionTokenOptions = {},
|
|
): Promise<string> {
|
|
return config.auth_token ?? (await resolveNotionAuthToken(config.auth_token_ref ?? '', options));
|
|
}
|
|
|
|
export async function notionConnectionToPullConfig(
|
|
config: KtxNotionConnectionConfig,
|
|
options: ResolveNotionTokenOptions = {},
|
|
): Promise<NotionPullConfig> {
|
|
const authToken = await resolveNotionConnectionAuthToken(config, options);
|
|
return notionPullConfigSchema.parse({
|
|
authToken,
|
|
crawlMode: config.crawl_mode,
|
|
rootPageIds: config.root_page_ids,
|
|
rootDatabaseIds: config.root_database_ids,
|
|
rootDataSourceIds: config.root_data_source_ids,
|
|
maxPagesPerRun: config.max_pages_per_run,
|
|
maxKnowledgeCreatesPerRun: config.max_knowledge_creates_per_run,
|
|
maxKnowledgeUpdatesPerRun: config.max_knowledge_updates_per_run,
|
|
lastSuccessfulCursor: null,
|
|
});
|
|
}
|